2024-11-13 19:43:03,418 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-13 19:43:03,451 main DEBUG Took 0.030395 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 19:43:03,451 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 19:43:03,451 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 19:43:03,452 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 19:43:03,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,460 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 19:43:03,478 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,480 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,481 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,483 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,484 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,486 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,486 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,487 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,488 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,489 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,491 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,492 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:43:03,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,494 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 19:43:03,495 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:43:03,499 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 19:43:03,502 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 19:43:03,502 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 19:43:03,503 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 19:43:03,504 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 19:43:03,515 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 19:43:03,518 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 19:43:03,519 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 19:43:03,520 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 19:43:03,520 main DEBUG createAppenders(={Console}) 2024-11-13 19:43:03,521 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-13 19:43:03,521 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-13 19:43:03,521 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-13 19:43:03,522 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 19:43:03,522 main DEBUG OutputStream closed 2024-11-13 19:43:03,522 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 19:43:03,522 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 19:43:03,523 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-13 19:43:03,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 19:43:03,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 19:43:03,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 19:43:03,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 19:43:03,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 19:43:03,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 19:43:03,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 19:43:03,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 19:43:03,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 19:43:03,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 19:43:03,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 19:43:03,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 19:43:03,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 19:43:03,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 19:43:03,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 19:43:03,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 19:43:03,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 19:43:03,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 19:43:03,624 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 19:43:03,624 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-13 19:43:03,624 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 19:43:03,625 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-13T19:43:04,034 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b 2024-11-13 19:43:04,038 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 19:43:04,039 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T19:43:04,051 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T19:43:04,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T19:43:04,107 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c, deleteOnExit=true 2024-11-13T19:43:04,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T19:43:04,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/test.cache.data in system properties and HBase conf 2024-11-13T19:43:04,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T19:43:04,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.log.dir in system properties and HBase conf 2024-11-13T19:43:04,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T19:43:04,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T19:43:04,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T19:43:04,252 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T19:43:04,405 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T19:43:04,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:43:04,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:43:04,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T19:43:04,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:43:04,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T19:43:04,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T19:43:04,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:43:04,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:43:04,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T19:43:04,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/nfs.dump.dir in system properties and HBase conf 2024-11-13T19:43:04,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/java.io.tmpdir in system properties and HBase conf 2024-11-13T19:43:04,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:43:04,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T19:43:04,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T19:43:05,403 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T19:43:05,538 INFO [Time-limited test {}] log.Log(170): Logging initialized @3213ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T19:43:05,653 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:43:05,739 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:43:05,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:43:05,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:43:05,777 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:43:05,791 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:43:05,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:43:05,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:43:06,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/java.io.tmpdir/jetty-localhost-40993-hadoop-hdfs-3_4_1-tests_jar-_-any-6250987917720783648/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:43:06,149 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:40993} 2024-11-13T19:43:06,150 INFO [Time-limited test {}] server.Server(415): Started @3825ms 2024-11-13T19:43:06,640 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:43:06,650 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:43:06,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:43:06,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:43:06,654 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:43:06,655 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:43:06,656 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:43:06,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/java.io.tmpdir/jetty-localhost-33833-hadoop-hdfs-3_4_1-tests_jar-_-any-16077137899329520592/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:43:06,799 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:33833} 2024-11-13T19:43:06,799 INFO [Time-limited test {}] server.Server(415): Started @4475ms 2024-11-13T19:43:06,867 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T19:43:07,543 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data1/current/BP-1564430831-172.17.0.2-1731526985115/current, will proceed with Du for space computation calculation, 2024-11-13T19:43:07,544 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data2/current/BP-1564430831-172.17.0.2-1731526985115/current, will proceed with Du for space computation calculation, 2024-11-13T19:43:07,596 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T19:43:07,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4fe70f0951b80de with lease ID 0xc1a356f67b7b9e00: Processing first storage report for DS-14d9dace-0fa2-4ef0-9aa0-a5193ead2148 from datanode DatanodeRegistration(127.0.0.1:38309, datanodeUuid=c0c64aab-1c9e-455b-90f3-92b4fe9dd18d, infoPort=33571, infoSecurePort=0, ipcPort=33689, storageInfo=lv=-57;cid=testClusterID;nsid=2064016488;c=1731526985115) 2024-11-13T19:43:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4fe70f0951b80de with lease ID 0xc1a356f67b7b9e00: from storage DS-14d9dace-0fa2-4ef0-9aa0-a5193ead2148 node DatanodeRegistration(127.0.0.1:38309, datanodeUuid=c0c64aab-1c9e-455b-90f3-92b4fe9dd18d, infoPort=33571, infoSecurePort=0, ipcPort=33689, storageInfo=lv=-57;cid=testClusterID;nsid=2064016488;c=1731526985115), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T19:43:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4fe70f0951b80de with lease ID 0xc1a356f67b7b9e00: Processing first storage report for DS-12ab9fd0-ae4e-4e1d-bcaf-8cb59cba373c from datanode DatanodeRegistration(127.0.0.1:38309, datanodeUuid=c0c64aab-1c9e-455b-90f3-92b4fe9dd18d, infoPort=33571, infoSecurePort=0, ipcPort=33689, storageInfo=lv=-57;cid=testClusterID;nsid=2064016488;c=1731526985115) 2024-11-13T19:43:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4fe70f0951b80de with lease ID 0xc1a356f67b7b9e00: from storage DS-12ab9fd0-ae4e-4e1d-bcaf-8cb59cba373c node DatanodeRegistration(127.0.0.1:38309, datanodeUuid=c0c64aab-1c9e-455b-90f3-92b4fe9dd18d, infoPort=33571, infoSecurePort=0, ipcPort=33689, storageInfo=lv=-57;cid=testClusterID;nsid=2064016488;c=1731526985115), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:43:07,711 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b 2024-11-13T19:43:07,822 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/zookeeper_0, clientPort=50411, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T19:43:07,836 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50411 2024-11-13T19:43:07,850 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:07,854 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741825_1001 (size=7) 2024-11-13T19:43:08,624 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e with version=8 2024-11-13T19:43:08,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/hbase-staging 2024-11-13T19:43:08,736 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T19:43:09,039 INFO [Time-limited test {}] client.ConnectionUtils(128): master/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:43:09,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,064 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:43:09,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:43:09,257 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T19:43:09,333 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T19:43:09,344 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T19:43:09,348 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:43:09,380 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4031 (auto-detected) 2024-11-13T19:43:09,381 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T19:43:09,404 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42037 2024-11-13T19:43:09,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42037 connecting to ZooKeeper ensemble=127.0.0.1:50411 2024-11-13T19:43:09,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420370x0, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:43:09,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42037-0x100eda004b50000 connected 2024-11-13T19:43:09,643 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:09,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:09,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:43:09,666 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e, hbase.cluster.distributed=false 2024-11-13T19:43:09,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:43:09,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-13T19:43:09,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42037 2024-11-13T19:43:09,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42037 2024-11-13T19:43:09,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-13T19:43:09,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42037 2024-11-13T19:43:09,897 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:43:09,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,900 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:43:09,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:43:09,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:43:09,904 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T19:43:09,908 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:43:09,911 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36227 2024-11-13T19:43:09,914 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36227 connecting to ZooKeeper ensemble=127.0.0.1:50411 2024-11-13T19:43:09,917 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:09,920 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:09,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362270x0, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:43:09,947 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:362270x0, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:43:09,953 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T19:43:09,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36227-0x100eda004b50001 connected 2024-11-13T19:43:09,965 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T19:43:09,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T19:43:09,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:43:09,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36227 2024-11-13T19:43:09,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36227 2024-11-13T19:43:09,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36227 2024-11-13T19:43:09,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36227 2024-11-13T19:43:10,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36227 2024-11-13T19:43:10,032 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;54d0784d1943:42037 2024-11-13T19:43:10,034 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/54d0784d1943,42037,1731526988795 2024-11-13T19:43:10,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:43:10,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:43:10,047 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/54d0784d1943,42037,1731526988795 2024-11-13T19:43:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T19:43:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:10,082 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T19:43:10,084 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/54d0784d1943,42037,1731526988795 from backup master directory 2024-11-13T19:43:10,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/54d0784d1943,42037,1731526988795 2024-11-13T19:43:10,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:43:10,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:43:10,089 WARN [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:43:10,089 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=54d0784d1943,42037,1731526988795 2024-11-13T19:43:10,092 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T19:43:10,094 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T19:43:10,167 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/hbase.id] with ID: abd38298-c95a-46fd-865b-a2bfc93c7871 2024-11-13T19:43:10,167 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/.tmp/hbase.id 2024-11-13T19:43:10,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741826_1002 (size=42) 2024-11-13T19:43:10,181 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/.tmp/hbase.id]:[hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/hbase.id] 2024-11-13T19:43:10,249 INFO [master/54d0784d1943:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:43:10,257 INFO [master/54d0784d1943:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T19:43:10,291 INFO [master/54d0784d1943:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 30ms. 2024-11-13T19:43:10,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:10,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:10,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741827_1003 (size=196) 2024-11-13T19:43:10,756 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:43:10,759 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T19:43:10,784 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:10,792 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:43:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741828_1004 (size=1189) 2024-11-13T19:43:11,357 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store 2024-11-13T19:43:11,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741829_1005 (size=34) 2024-11-13T19:43:11,864 INFO [master/54d0784d1943:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T19:43:11,867 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:11,869 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:43:11,869 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:43:11,869 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:43:11,871 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:43:11,871 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:43:11,871 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:43:11,872 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731526991869Disabling compacts and flushes for region at 1731526991869Disabling writes for close at 1731526991871 (+2 ms)Writing region close event to WAL at 1731526991871Closed at 1731526991871 2024-11-13T19:43:11,875 WARN [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/.initializing 2024-11-13T19:43:11,875 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/WALs/54d0784d1943,42037,1731526988795 2024-11-13T19:43:11,884 INFO [master/54d0784d1943:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:43:11,910 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C42037%2C1731526988795, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/WALs/54d0784d1943,42037,1731526988795, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/oldWALs, maxLogs=10 2024-11-13T19:43:11,946 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/WALs/54d0784d1943,42037,1731526988795/54d0784d1943%2C42037%2C1731526988795.1731526991916, exclude list is [], retry=0 2024-11-13T19:43:11,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38309,DS-14d9dace-0fa2-4ef0-9aa0-a5193ead2148,DISK] 2024-11-13T19:43:11,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T19:43:12,033 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/WALs/54d0784d1943,42037,1731526988795/54d0784d1943%2C42037%2C1731526988795.1731526991916 2024-11-13T19:43:12,034 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33571:33571)] 2024-11-13T19:43:12,035 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:43:12,035 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:12,039 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,040 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T19:43:12,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:12,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:12,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T19:43:12,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:12,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:12,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T19:43:12,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:12,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:12,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T19:43:12,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:12,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:12,163 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,173 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,175 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,182 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,182 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,186 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T19:43:12,190 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:43:12,195 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:43:12,196 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62799621, jitterRate=-0.06421272456645966}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T19:43:12,203 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731526992059Initializing all the Stores at 1731526992062 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526992062Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526992063 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526992063Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526992063Cleaning up temporary data from old regions at 1731526992182 (+119 ms)Region opened successfully at 1731526992203 (+21 ms) 2024-11-13T19:43:12,207 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T19:43:12,248 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4dfc4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:43:12,294 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T19:43:12,311 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T19:43:12,311 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T19:43:12,316 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T19:43:12,318 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-13T19:43:12,325 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T19:43:12,325 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T19:43:12,401 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T19:43:12,434 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T19:43:12,438 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T19:43:12,441 INFO [master/54d0784d1943:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T19:43:12,446 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T19:43:12,451 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T19:43:12,463 INFO [master/54d0784d1943:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T19:43:12,486 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T19:43:12,497 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T19:43:12,505 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T19:43:12,515 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T19:43:12,546 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T19:43:12,548 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T19:43:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:43:12,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:43:12,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,561 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=54d0784d1943,42037,1731526988795, sessionid=0x100eda004b50000, setting cluster-up flag (Was=false) 2024-11-13T19:43:12,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,601 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T19:43:12,622 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54d0784d1943,42037,1731526988795 2024-11-13T19:43:12,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:12,648 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T19:43:12,651 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54d0784d1943,42037,1731526988795 2024-11-13T19:43:12,659 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T19:43:12,732 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(746): ClusterId : abd38298-c95a-46fd-865b-a2bfc93c7871 2024-11-13T19:43:12,735 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T19:43:12,741 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T19:43:12,741 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T19:43:12,745 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T19:43:12,746 DEBUG [RS:0;54d0784d1943:36227 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2912367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:43:12,753 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T19:43:12,763 INFO [master/54d0784d1943:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T19:43:12,769 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;54d0784d1943:36227 2024-11-13T19:43:12,771 INFO [master/54d0784d1943:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T19:43:12,774 INFO [RS:0;54d0784d1943:36227 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T19:43:12,774 INFO [RS:0;54d0784d1943:36227 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T19:43:12,774 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T19:43:12,778 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,42037,1731526988795 with port=36227, startcode=1731526989835 2024-11-13T19:43:12,778 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 54d0784d1943,42037,1731526988795 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T19:43:12,785 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:43:12,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:43:12,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:43:12,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:43:12,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/54d0784d1943:0, corePoolSize=10, maxPoolSize=10 2024-11-13T19:43:12,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,787 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:43:12,787 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,792 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:43:12,792 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T19:43:12,794 DEBUG [RS:0;54d0784d1943:36227 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T19:43:12,797 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731527022797 2024-11-13T19:43:12,799 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T19:43:12,800 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T19:43:12,801 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:12,801 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T19:43:12,805 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T19:43:12,805 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T19:43:12,805 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T19:43:12,806 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T19:43:12,810 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,817 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T19:43:12,819 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T19:43:12,819 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T19:43:12,824 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T19:43:12,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741831_1007 (size=1321) 2024-11-13T19:43:12,827 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T19:43:12,831 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731526992830,5,FailOnTimeoutGroup] 2024-11-13T19:43:12,832 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T19:43:12,833 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:43:12,833 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731526992831,5,FailOnTimeoutGroup] 2024-11-13T19:43:12,833 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,834 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T19:43:12,838 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,839 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T19:43:12,861 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42037 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 54d0784d1943,36227,1731526989835 2024-11-13T19:43:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741832_1008 (size=32) 2024-11-13T19:43:12,869 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42037 {}] master.ServerManager(517): Registering regionserver=54d0784d1943,36227,1731526989835 2024-11-13T19:43:12,889 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:43:12,889 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34445 2024-11-13T19:43:12,890 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T19:43:12,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:43:12,896 DEBUG [RS:0;54d0784d1943:36227 {}] zookeeper.ZKUtil(111): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54d0784d1943,36227,1731526989835 2024-11-13T19:43:12,896 WARN [RS:0;54d0784d1943:36227 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:43:12,896 INFO [RS:0;54d0784d1943:36227 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:43:12,897 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835 2024-11-13T19:43:12,913 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54d0784d1943,36227,1731526989835] 2024-11-13T19:43:12,952 INFO [RS:0;54d0784d1943:36227 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T19:43:12,974 INFO [RS:0;54d0784d1943:36227 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T19:43:12,982 INFO [RS:0;54d0784d1943:36227 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:43:12,982 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,983 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T19:43:12,992 INFO [RS:0;54d0784d1943:36227 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T19:43:12,994 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:12,995 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,995 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,995 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,996 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,996 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,996 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:43:12,996 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,996 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,997 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,997 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,997 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,998 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:43:12,998 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:43:12,998 DEBUG [RS:0;54d0784d1943:36227 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:43:13,009 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,014 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,014 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,014 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,014 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,014 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,36227,1731526989835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:43:13,047 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T19:43:13,050 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,36227,1731526989835-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,050 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,051 INFO [RS:0;54d0784d1943:36227 {}] regionserver.Replication(171): 54d0784d1943,36227,1731526989835 started 2024-11-13T19:43:13,082 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:13,082 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1482): Serving as 54d0784d1943,36227,1731526989835, RpcServer on 54d0784d1943/172.17.0.2:36227, sessionid=0x100eda004b50001 2024-11-13T19:43:13,083 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T19:43:13,084 DEBUG [RS:0;54d0784d1943:36227 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54d0784d1943,36227,1731526989835 2024-11-13T19:43:13,084 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,36227,1731526989835' 2024-11-13T19:43:13,084 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T19:43:13,086 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T19:43:13,087 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T19:43:13,087 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T19:43:13,087 DEBUG [RS:0;54d0784d1943:36227 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54d0784d1943,36227,1731526989835 2024-11-13T19:43:13,087 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,36227,1731526989835' 2024-11-13T19:43:13,087 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T19:43:13,088 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T19:43:13,089 DEBUG [RS:0;54d0784d1943:36227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T19:43:13,089 INFO [RS:0;54d0784d1943:36227 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T19:43:13,089 INFO [RS:0;54d0784d1943:36227 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T19:43:13,197 INFO [RS:0;54d0784d1943:36227 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:43:13,202 INFO [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C36227%2C1731526989835, suffix=, logDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/oldWALs, maxLogs=32 2024-11-13T19:43:13,230 DEBUG [RS:0;54d0784d1943:36227 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835/54d0784d1943%2C36227%2C1731526989835.1731526993206, exclude list is [], retry=0 2024-11-13T19:43:13,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38309,DS-14d9dace-0fa2-4ef0-9aa0-a5193ead2148,DISK] 2024-11-13T19:43:13,248 INFO [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835/54d0784d1943%2C36227%2C1731526989835.1731526993206 2024-11-13T19:43:13,249 DEBUG [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33571:33571)] 2024-11-13T19:43:13,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:13,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:43:13,274 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:43:13,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:43:13,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:43:13,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:43:13,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:43:13,285 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:43:13,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:43:13,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:43:13,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740 2024-11-13T19:43:13,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740 2024-11-13T19:43:13,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:43:13,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:43:13,322 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:43:13,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:43:13,338 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:43:13,340 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71127205, jitterRate=0.059877946972846985}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:43:13,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731526993267Initializing all the Stores at 1731526993269 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993269Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993270 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526993270Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993270Cleaning up temporary data from old regions at 1731526993321 (+51 ms)Region opened successfully at 1731526993343 (+22 ms) 2024-11-13T19:43:13,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:43:13,344 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:43:13,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:43:13,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T19:43:13,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:43:13,350 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:43:13,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731526993343Disabling compacts and flushes for region at 1731526993343Disabling writes for close at 1731526993344 (+1 ms)Writing region close event to WAL at 1731526993349 (+5 ms)Closed at 1731526993350 (+1 ms) 2024-11-13T19:43:13,355 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:43:13,355 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T19:43:13,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T19:43:13,380 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:43:13,384 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T19:43:13,538 DEBUG [54d0784d1943:42037 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T19:43:13,556 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:13,566 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54d0784d1943,36227,1731526989835, state=OPENING 2024-11-13T19:43:13,586 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T19:43:13,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:13,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:43:13,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:43:13,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:43:13,616 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:43:13,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:13,809 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:43:13,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53157, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:43:13,827 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T19:43:13,828 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:43:13,828 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T19:43:13,833 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C36227%2C1731526989835.meta, suffix=.meta, logDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835, archiveDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/oldWALs, maxLogs=32 2024-11-13T19:43:13,857 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835/54d0784d1943%2C36227%2C1731526989835.meta.1731526993835.meta, exclude list is [], retry=0 2024-11-13T19:43:13,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38309,DS-14d9dace-0fa2-4ef0-9aa0-a5193ead2148,DISK] 2024-11-13T19:43:13,871 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/WALs/54d0784d1943,36227,1731526989835/54d0784d1943%2C36227%2C1731526989835.meta.1731526993835.meta 2024-11-13T19:43:13,872 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33571:33571)] 2024-11-13T19:43:13,872 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:43:13,875 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T19:43:13,879 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T19:43:13,885 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T19:43:13,891 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T19:43:13,891 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:13,892 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T19:43:13,892 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T19:43:13,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:43:13,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:43:13,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:43:13,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:43:13,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:43:13,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:43:13,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:43:13,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:43:13,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:13,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:43:13,915 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:43:13,917 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740 2024-11-13T19:43:13,924 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740 2024-11-13T19:43:13,927 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:43:13,927 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:43:13,928 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:43:13,932 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:43:13,934 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70867384, jitterRate=0.05600631237030029}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:43:13,934 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T19:43:13,937 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731526993892Writing region info on filesystem at 1731526993893 (+1 ms)Initializing all the Stores at 1731526993895 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993895Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993896 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526993896Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731526993896Cleaning up temporary data from old regions at 1731526993927 (+31 ms)Running coprocessor post-open hooks at 1731526993934 (+7 ms)Region opened successfully at 1731526993937 (+3 ms) 2024-11-13T19:43:13,947 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731526993798 2024-11-13T19:43:13,963 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T19:43:13,964 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T19:43:13,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:13,969 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54d0784d1943,36227,1731526989835, state=OPEN 2024-11-13T19:43:13,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:43:13,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:43:13,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:43:13,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:43:13,976 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:13,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T19:43:13,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=54d0784d1943,36227,1731526989835 in 357 msec 2024-11-13T19:43:13,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T19:43:13,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 626 msec 2024-11-13T19:43:13,997 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:43:13,997 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T19:43:14,028 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:14,029 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:14,055 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:14,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:14,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3920 sec 2024-11-13T19:43:14,088 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731526994088, completionTime=-1 2024-11-13T19:43:14,091 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T19:43:14,091 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T19:43:14,123 INFO [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T19:43:14,123 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731527054123 2024-11-13T19:43:14,123 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731527114123 2024-11-13T19:43:14,123 INFO [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-13T19:43:14,129 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,129 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,130 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,132 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-54d0784d1943:42037, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,132 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,133 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,142 DEBUG [master/54d0784d1943:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T19:43:14,175 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.086sec 2024-11-13T19:43:14,177 INFO [master/54d0784d1943:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T19:43:14,179 INFO [master/54d0784d1943:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T19:43:14,180 INFO [master/54d0784d1943:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T19:43:14,181 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T19:43:14,181 INFO [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T19:43:14,182 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:43:14,183 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T19:43:14,195 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T19:43:14,196 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T19:43:14,196 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,42037,1731526988795-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:43:14,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:14,251 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T19:43:14,251 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T19:43:14,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:14,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:14,283 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0d6658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:14,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:14,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:14,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:14,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:14,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=54d0784d1943,42037,1731526988795 2024-11-13T19:43:14,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=610, ProcessCount=11, AvailableMemoryMB=1448 2024-11-13T19:43:14,369 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:43:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:43:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@603c91f2 2024-11-13T19:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:43:14,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:43:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:43:14,421 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:43:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:14,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:43:14,440 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:14,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T19:43:14,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:43:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:43:14,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741835_1011 (size=1061) 2024-11-13T19:43:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:43:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:43:14,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:43:14,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741836_1012 (size=53) 2024-11-13T19:43:14,940 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:14,940 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 31328a0914442b3583375df8a5770e39, disabling compactions & flushes 2024-11-13T19:43:14,940 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:14,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:14,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. after waiting 0 ms 2024-11-13T19:43:14,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:14,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:14,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 31328a0914442b3583375df8a5770e39: Waiting for close lock at 1731526994940Disabling compacts and flushes for region at 1731526994940Disabling writes for close at 1731526994941 (+1 ms)Writing region close event to WAL at 1731526994941Closed at 1731526994941 2024-11-13T19:43:14,947 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:43:14,955 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731526994947"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731526994947"}]},"ts":"1731526994947"} 2024-11-13T19:43:14,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:43:14,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:43:14,966 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731526994964"}]},"ts":"1731526994964"} 2024-11-13T19:43:14,971 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:43:14,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, ASSIGN}] 2024-11-13T19:43:14,975 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, ASSIGN 2024-11-13T19:43:14,977 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:43:15,130 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=31328a0914442b3583375df8a5770e39, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:15,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, ASSIGN because future has completed 2024-11-13T19:43:15,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:15,309 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:15,310 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:43:15,310 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,311 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:15,311 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,311 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,321 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,326 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:15,327 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31328a0914442b3583375df8a5770e39 columnFamilyName A 2024-11-13T19:43:15,327 DEBUG [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:15,329 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(327): Store=31328a0914442b3583375df8a5770e39/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:15,330 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,334 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:15,335 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31328a0914442b3583375df8a5770e39 columnFamilyName B 2024-11-13T19:43:15,335 DEBUG [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:15,336 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(327): Store=31328a0914442b3583375df8a5770e39/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:15,336 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,343 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:15,345 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31328a0914442b3583375df8a5770e39 columnFamilyName C 2024-11-13T19:43:15,346 DEBUG [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:15,349 INFO [StoreOpener-31328a0914442b3583375df8a5770e39-1 {}] regionserver.HStore(327): Store=31328a0914442b3583375df8a5770e39/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:15,349 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:15,350 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,352 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,352 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,355 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,355 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,356 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:43:15,360 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,369 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:43:15,372 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 31328a0914442b3583375df8a5770e39; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61979664, jitterRate=-0.0764310359954834}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:43:15,372 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:15,373 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 31328a0914442b3583375df8a5770e39: Running coprocessor pre-open hook at 1731526995311Writing region info on filesystem at 1731526995311Initializing all the Stores at 1731526995314 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526995315 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526995321 (+6 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731526995321Cleaning up temporary data from old regions at 1731526995355 (+34 ms)Running coprocessor post-open hooks at 1731526995372 (+17 ms)Region opened successfully at 1731526995373 (+1 ms) 2024-11-13T19:43:15,377 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., pid=6, masterSystemTime=1731526995292 2024-11-13T19:43:15,383 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:15,383 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:15,385 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=31328a0914442b3583375df8a5770e39, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:15,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:43:15,398 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42037 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=54d0784d1943,36227,1731526989835, table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T19:43:15,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T19:43:15,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 in 264 msec 2024-11-13T19:43:15,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T19:43:15,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, ASSIGN in 433 msec 2024-11-13T19:43:15,416 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:43:15,416 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731526995416"}]},"ts":"1731526995416"} 2024-11-13T19:43:15,421 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:43:15,423 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:43:15,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 998 msec 2024-11-13T19:43:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:43:15,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:15,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9032552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,630 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665358ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b26fa3 2024-11-13T19:43:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:43:15,647 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:43:15,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776dd602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0238d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,668 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ac9255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,700 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec20ccb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,718 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,731 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b8f587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,739 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592e422b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69964181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25cbae8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,750 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11dd0198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cff1959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eee02d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec64ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e58d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d07518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,775 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c24f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d1d32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:15,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:15,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2341aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:15,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fb5a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:15,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:15,807 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T19:43:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,811 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:15,812 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:15,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:15,822 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,830 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,843 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,845 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,847 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,858 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,873 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:15,875 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,876 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:43:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:15,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:43:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:16,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:16,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:16,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:16,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:16,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:16,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:16,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/00b84d70ae004d4b877a49a4f9a77946 is 50, key is test_row_0/A:col10/1731526995920/Put/seqid=0 2024-11-13T19:43:16,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741837_1013 (size=12001) 2024-11-13T19:43:16,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/00b84d70ae004d4b877a49a4f9a77946 2024-11-13T19:43:16,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526996063, max timestamp=1731526996334Num uniques 859; 2024-11-13T19:43:16,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:16,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731526996063, max timestamp=1731526996334Num uniques 859; 2024-11-13T19:43:16,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:43:16,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526996063, max timestamp=1731526996339Num uniques 859; 2024-11-13T19:43:16,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:16,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:16,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cedb0ddc68d74a688f10d0892304ca4f is 50, key is test_row_0/B:col10/1731526995920/Put/seqid=0 2024-11-13T19:43:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741838_1014 (size=12001) 2024-11-13T19:43:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527056502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527056502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527056509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527056542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527056544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527056719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527056720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527056721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527056722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527056721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cedb0ddc68d74a688f10d0892304ca4f 2024-11-13T19:43:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527056939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527056951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527056951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:16,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527056954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527056955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:16,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:17,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ffe3aeb5c63e469d9ad5d778c7b31d03 is 50, key is test_row_0/C:col10/1731526995920/Put/seqid=0 2024-11-13T19:43:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741839_1015 (size=12001) 2024-11-13T19:43:17,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ffe3aeb5c63e469d9ad5d778c7b31d03 2024-11-13T19:43:17,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/00b84d70ae004d4b877a49a4f9a77946 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946 2024-11-13T19:43:17,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:43:17,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cedb0ddc68d74a688f10d0892304ca4f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f 2024-11-13T19:43:17,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:43:17,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ffe3aeb5c63e469d9ad5d778c7b31d03 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03 2024-11-13T19:43:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:43:17,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for 31328a0914442b3583375df8a5770e39 in 1162ms, sequenceid=12, compaction requested=false 2024-11-13T19:43:17,183 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:43:17,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:17,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731526996443, max timestamp=1731526997273Num uniques 1018; 2024-11-13T19:43:17,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T19:43:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:17,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=102.48 KB heapSize=291.29 KB 2024-11-13T19:43:17,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:17,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731526996443, max timestamp=1731526997273Num uniques 1018; 2024-11-13T19:43:17,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T19:43:17,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:17,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:17,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:43:17,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10c508062930424a9679774c964ddf9c is 50, key is test_row_0/A:col10/1731526996518/Put/seqid=0 2024-11-13T19:43:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741840_1016 (size=12001) 2024-11-13T19:43:17,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10c508062930424a9679774c964ddf9c 2024-11-13T19:43:17,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f546f3533be4f5eb7a9c6e0e7a78f84 is 50, key is test_row_0/B:col10/1731526996518/Put/seqid=0 2024-11-13T19:43:17,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741841_1017 (size=12001) 2024-11-13T19:43:17,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f546f3533be4f5eb7a9c6e0e7a78f84 2024-11-13T19:43:17,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:17,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:17,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/aeea3c3f37f44c0cab8478fef1fa6199 is 50, key is test_row_0/C:col10/1731526996518/Put/seqid=0 2024-11-13T19:43:17,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527057451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527057452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527057457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741842_1018 (size=12001) 2024-11-13T19:43:17,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527057489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/aeea3c3f37f44c0cab8478fef1fa6199 2024-11-13T19:43:17,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527057478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10c508062930424a9679774c964ddf9c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c 2024-11-13T19:43:17,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T19:43:17,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f546f3533be4f5eb7a9c6e0e7a78f84 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84 2024-11-13T19:43:17,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T19:43:17,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/aeea3c3f37f44c0cab8478fef1fa6199 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199 2024-11-13T19:43:17,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527057622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:17,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T19:43:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527057623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:17,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 351ms, sequenceid=50, compaction requested=false 2024-11-13T19:43:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:17,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:43:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:17,654 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-13T19:43:17,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526997450, max timestamp=1731526997645Num uniques 859; 2024-11-13T19:43:17,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:17,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526997450, max timestamp=1731526997645Num uniques 859; 2024-11-13T19:43:17,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c668c37816a840a1ad6c0bdbddf0c5a6 is 50, key is test_row_0/A:col10/1731526997645/Put/seqid=0 2024-11-13T19:43:17,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741843_1019 (size=14341) 2024-11-13T19:43:17,809 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c668c37816a840a1ad6c0bdbddf0c5a6 2024-11-13T19:43:17,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6235d6dec3ac40b7a4b20f8fb88a87d7 is 50, key is test_row_0/B:col10/1731526997645/Put/seqid=0 2024-11-13T19:43:17,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741844_1020 (size=12001) 2024-11-13T19:43:17,917 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6235d6dec3ac40b7a4b20f8fb88a87d7 2024-11-13T19:43:17,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/e01d3715deea4416a99930d207a34e83 is 50, key is test_row_0/C:col10/1731526997645/Put/seqid=0 2024-11-13T19:43:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:18,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526997955, max timestamp=1731526997988Num uniques 859; 2024-11-13T19:43:18,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:18,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741845_1021 (size=12001) 2024-11-13T19:43:18,023 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/e01d3715deea4416a99930d207a34e83 2024-11-13T19:43:18,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526997955, max timestamp=1731526997990Num uniques 859; 2024-11-13T19:43:18,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c668c37816a840a1ad6c0bdbddf0c5a6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6 2024-11-13T19:43:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6, entries=200, sequenceid=71, filesize=14.0 K 2024-11-13T19:43:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6235d6dec3ac40b7a4b20f8fb88a87d7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7 2024-11-13T19:43:18,092 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7, entries=150, sequenceid=71, filesize=11.7 K 2024-11-13T19:43:18,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/e01d3715deea4416a99930d207a34e83 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83 2024-11-13T19:43:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527058060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527058064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,111 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83, entries=150, sequenceid=71, filesize=11.7 K 2024-11-13T19:43:18,113 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=125.23 KB/128240 for 31328a0914442b3583375df8a5770e39 in 460ms, sequenceid=71, compaction requested=true 2024-11-13T19:43:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T19:43:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T19:43:18,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:18,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T19:43:18,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2970 sec 2024-11-13T19:43:18,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/674218d886e44343bd9206201d4ec695 is 50, key is test_row_0/A:col10/1731526998089/Put/seqid=0 2024-11-13T19:43:18,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3570 sec 2024-11-13T19:43:18,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741846_1022 (size=14341) 2024-11-13T19:43:18,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/674218d886e44343bd9206201d4ec695 2024-11-13T19:43:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527058205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527058217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527058242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527058251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527058256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/69d89a006bc9489b9dae225ba1a6156f is 50, key is test_row_0/B:col10/1731526998089/Put/seqid=0 2024-11-13T19:43:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527058369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527058374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741847_1023 (size=12001) 2024-11-13T19:43:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527058384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527058385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527058458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527058585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527058595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527058634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527058645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/69d89a006bc9489b9dae225ba1a6156f 2024-11-13T19:43:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527058793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/88cb8973cc534cc69ffe7b529d8205a3 is 50, key is test_row_0/C:col10/1731526998089/Put/seqid=0 2024-11-13T19:43:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741848_1024 (size=12001) 2024-11-13T19:43:18,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/88cb8973cc534cc69ffe7b529d8205a3 2024-11-13T19:43:18,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527058918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527058906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/674218d886e44343bd9206201d4ec695 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695 2024-11-13T19:43:18,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527058974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:18,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695, entries=200, sequenceid=103, filesize=14.0 K 2024-11-13T19:43:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527058984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/69d89a006bc9489b9dae225ba1a6156f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f 2024-11-13T19:43:19,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f, entries=150, sequenceid=103, filesize=11.7 K 2024-11-13T19:43:19,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/88cb8973cc534cc69ffe7b529d8205a3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3 2024-11-13T19:43:19,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3, entries=150, sequenceid=103, filesize=11.7 K 2024-11-13T19:43:19,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=53.67 KB/54960 for 31328a0914442b3583375df8a5770e39 in 927ms, sequenceid=103, compaction requested=true 2024-11-13T19:43:19,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:19,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:19,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:19,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:19,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:19,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:19,089 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:19,093 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:19,097 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:19,099 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:19,100 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:19,102 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:19,103 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:19,103 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:19,105 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=51.4 K 2024-11-13T19:43:19,105 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=46.9 K 2024-11-13T19:43:19,107 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b84d70ae004d4b877a49a4f9a77946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731526995920 2024-11-13T19:43:19,107 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cedb0ddc68d74a688f10d0892304ca4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731526995920 2024-11-13T19:43:19,108 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c508062930424a9679774c964ddf9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731526996495 2024-11-13T19:43:19,111 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f546f3533be4f5eb7a9c6e0e7a78f84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731526996495 2024-11-13T19:43:19,111 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c668c37816a840a1ad6c0bdbddf0c5a6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731526997450 2024-11-13T19:43:19,132 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674218d886e44343bd9206201d4ec695, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731526998060 2024-11-13T19:43:19,132 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6235d6dec3ac40b7a4b20f8fb88a87d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731526997469 2024-11-13T19:43:19,133 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 69d89a006bc9489b9dae225ba1a6156f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731526998060 2024-11-13T19:43:19,214 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#13 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:19,215 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/9ff15312f661405da03f18398b0226a4 is 50, key is test_row_0/A:col10/1731526998089/Put/seqid=0 2024-11-13T19:43:19,217 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#12 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:19,218 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/f0afbed517244edab56c157d305134a3 is 50, key is test_row_0/B:col10/1731526998089/Put/seqid=0 2024-11-13T19:43:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741849_1025 (size=12139) 2024-11-13T19:43:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741850_1026 (size=12139) 2024-11-13T19:43:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:19,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:43:19,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:19,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:19,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:19,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:19,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:19,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:19,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/98026d98593b421d804e45d71abaac60 is 50, key is test_row_0/A:col10/1731526998223/Put/seqid=0 2024-11-13T19:43:19,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:43:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741851_1027 (size=12001) 2024-11-13T19:43:19,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/98026d98593b421d804e45d71abaac60 2024-11-13T19:43:19,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e267b7f8cb6e48ada4f7a26343f9e72f is 50, key is test_row_0/B:col10/1731526998223/Put/seqid=0 2024-11-13T19:43:19,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741852_1028 (size=12001) 2024-11-13T19:43:19,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e267b7f8cb6e48ada4f7a26343f9e72f 2024-11-13T19:43:19,514 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T19:43:19,516 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T19:43:19,517 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T19:43:19,519 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:43:19,519 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T19:43:19,519 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T19:43:19,519 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T19:43:19,520 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:43:19,520 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T19:43:19,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8fdeee9a71204fd1b18ff8767f094ec7 is 50, key is test_row_0/C:col10/1731526998223/Put/seqid=0 2024-11-13T19:43:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527059511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527059515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527059524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527059529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527059529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741853_1029 (size=12001) 2024-11-13T19:43:19,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8fdeee9a71204fd1b18ff8767f094ec7 2024-11-13T19:43:19,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527059649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527059651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527059650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/98026d98593b421d804e45d71abaac60 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60 2024-11-13T19:43:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527059659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527059670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:19,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T19:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e267b7f8cb6e48ada4f7a26343f9e72f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f 2024-11-13T19:43:19,729 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/9ff15312f661405da03f18398b0226a4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/9ff15312f661405da03f18398b0226a4 2024-11-13T19:43:19,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T19:43:19,735 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/f0afbed517244edab56c157d305134a3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/f0afbed517244edab56c157d305134a3 2024-11-13T19:43:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8fdeee9a71204fd1b18ff8767f094ec7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7 2024-11-13T19:43:19,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T19:43:19,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 437ms, sequenceid=115, compaction requested=true 2024-11-13T19:43:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:19,785 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into f0afbed517244edab56c157d305134a3(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:19,785 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:43:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:43:19,788 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=12, startTime=1731526999086; duration=0sec 2024-11-13T19:43:19,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:43:19,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:19,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:43:19,793 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 9ff15312f661405da03f18398b0226a4(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:19,794 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:19,794 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=12, startTime=1731526999064; duration=0sec 2024-11-13T19:43:19,794 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:43:19,794 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:19,798 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:43:19,798 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:19,798 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:19,799 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=58.6 K 2024-11-13T19:43:19,800 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe3aeb5c63e469d9ad5d778c7b31d03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731526995920 2024-11-13T19:43:19,803 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting aeea3c3f37f44c0cab8478fef1fa6199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731526996495 2024-11-13T19:43:19,805 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-13T19:43:19,805 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:19,806 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:19,806 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:19,806 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:19,806 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:43:19,807 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e01d3715deea4416a99930d207a34e83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731526997469 2024-11-13T19:43:19,808 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 88cb8973cc534cc69ffe7b529d8205a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731526998060 2024-11-13T19:43:19,809 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdeee9a71204fd1b18ff8767f094ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731526998194 2024-11-13T19:43:19,815 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:19,815 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:19,816 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:19,818 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:19,818 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:43:19,821 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:19,821 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:19,821 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:19,821 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:19,842 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#17 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:19,843 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/77cabdb8330d4914b366b7f97f2033db is 50, key is test_row_0/C:col10/1731526998223/Put/seqid=0 2024-11-13T19:43:19,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526999519, max timestamp=1731526999882Num uniques 859; 2024-11-13T19:43:19,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526999519, max timestamp=1731526999882Num uniques 859; 2024-11-13T19:43:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731526999519, max timestamp=1731526999882Num uniques 859; 2024-11-13T19:43:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:19,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741854_1030 (size=12173) 2024-11-13T19:43:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:19,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-13T19:43:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:19,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:19,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:19,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:19,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:19,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:19,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1ba3656721514434a24cbf595116206b is 50, key is test_row_0/A:col10/1731526999880/Put/seqid=0 2024-11-13T19:43:19,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741855_1031 (size=12151) 2024-11-13T19:43:19,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1ba3656721514434a24cbf595116206b 2024-11-13T19:43:20,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731526999962, max timestamp=1731527000000Num uniques 859; 2024-11-13T19:43:20,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:20,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731526999962, max timestamp=1731527000000Num uniques 859; 2024-11-13T19:43:20,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:43:20,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:20,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f4aa884554f4c01b5a73a3c8107d006 is 50, key is test_row_0/B:col10/1731526999880/Put/seqid=0 2024-11-13T19:43:20,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:20,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731526999962, max timestamp=1731527000000Num uniques 859; 2024-11-13T19:43:20,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T19:43:20,037 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:20,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:20,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:20,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741856_1032 (size=12151) 2024-11-13T19:43:20,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f4aa884554f4c01b5a73a3c8107d006 2024-11-13T19:43:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527060071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527060093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/69305634d70d48b29aa87842b37993df is 50, key is test_row_0/C:col10/1731526999880/Put/seqid=0 2024-11-13T19:43:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527060106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527060114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527060122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:20,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741857_1033 (size=12151) 2024-11-13T19:43:20,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/69305634d70d48b29aa87842b37993df 2024-11-13T19:43:20,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1ba3656721514434a24cbf595116206b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b 2024-11-13T19:43:20,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T19:43:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/0f4aa884554f4c01b5a73a3c8107d006 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006 2024-11-13T19:43:20,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T19:43:20,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:20,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/69305634d70d48b29aa87842b37993df as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df 2024-11-13T19:43:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:20,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T19:43:20,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for 31328a0914442b3583375df8a5770e39 in 342ms, sequenceid=143, compaction requested=true 2024-11-13T19:43:20,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:20,263 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:20,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:20,266 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:20,266 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:20,266 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,266 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/9ff15312f661405da03f18398b0226a4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=35.4 K 2024-11-13T19:43:20,267 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff15312f661405da03f18398b0226a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731526998060 2024-11-13T19:43:20,268 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98026d98593b421d804e45d71abaac60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731526998194 2024-11-13T19:43:20,269 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba3656721514434a24cbf595116206b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731526999880 2024-11-13T19:43:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:20,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527000084, max timestamp=1731527000277Num uniques 1068; 2024-11-13T19:43:20,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:20,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=105.11 KB heapSize=292.59 KB 2024-11-13T19:43:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527000084, max timestamp=1731527000277Num uniques 1068; 2024-11-13T19:43:20,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:20,288 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:20,289 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/210d471d9b29485593fa4194c94a8c12 is 50, key is test_row_0/A:col10/1731526999880/Put/seqid=0 2024-11-13T19:43:20,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12263, totalHeapSize=28112, min timestamp=1731527000084, max timestamp=1731527000277Num uniques 1068; 2024-11-13T19:43:20,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48903, new segment data size=12263, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:20,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:20,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/543cda9f746742cfb9518f9438317263 is 50, key is test_row_0/A:col10/1731527000091/Put/seqid=0 2024-11-13T19:43:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741858_1034 (size=12391) 2024-11-13T19:43:20,341 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/77cabdb8330d4914b366b7f97f2033db as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/77cabdb8330d4914b366b7f97f2033db 2024-11-13T19:43:20,354 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/210d471d9b29485593fa4194c94a8c12 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/210d471d9b29485593fa4194c94a8c12 2024-11-13T19:43:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741859_1035 (size=12151) 2024-11-13T19:43:20,371 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 77cabdb8330d4914b366b7f97f2033db(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:20,371 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:20,371 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=11, startTime=1731526999786; duration=0sec 2024-11-13T19:43:20,372 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:20,372 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:20,372 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:20,372 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:20,375 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 210d471d9b29485593fa4194c94a8c12(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:20,375 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:20,376 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527000262; duration=0sec 2024-11-13T19:43:20,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/543cda9f746742cfb9518f9438317263 2024-11-13T19:43:20,376 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:20,377 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:20,377 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:43:20,379 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:20,379 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:20,379 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,379 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/f0afbed517244edab56c157d305134a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=35.4 K 2024-11-13T19:43:20,381 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f0afbed517244edab56c157d305134a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731526998060 2024-11-13T19:43:20,382 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e267b7f8cb6e48ada4f7a26343f9e72f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731526998194 2024-11-13T19:43:20,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:20,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:20,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:20,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:20,383 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4aa884554f4c01b5a73a3c8107d006, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731526999880 2024-11-13T19:43:20,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/a54062f9a6494755971251dcebd6066e is 50, key is test_row_0/B:col10/1731527000091/Put/seqid=0 2024-11-13T19:43:20,419 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:20,420 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6c80cb22746a4f2bab9f0f40dc67ac83 is 50, key is test_row_0/B:col10/1731526999880/Put/seqid=0 2024-11-13T19:43:20,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527000320, max timestamp=1731527000426Num uniques 859; 2024-11-13T19:43:20,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527000320, max timestamp=1731527000426Num uniques 859; 2024-11-13T19:43:20,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:20,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:20,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527000320, max timestamp=1731527000426Num uniques 859; 2024-11-13T19:43:20,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:20,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741860_1036 (size=12151) 2024-11-13T19:43:20,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/a54062f9a6494755971251dcebd6066e 2024-11-13T19:43:20,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741861_1037 (size=12391) 2024-11-13T19:43:20,489 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6c80cb22746a4f2bab9f0f40dc67ac83 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6c80cb22746a4f2bab9f0f40dc67ac83 2024-11-13T19:43:20,504 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 6c80cb22746a4f2bab9f0f40dc67ac83(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:20,504 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:20,504 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527000263; duration=0sec 2024-11-13T19:43:20,504 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:20,504 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:20,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527000466, max timestamp=1731527000502Num uniques 1018; 2024-11-13T19:43:20,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3780312960934d27897c643908459186 is 50, key is test_row_0/C:col10/1731527000091/Put/seqid=0 2024-11-13T19:43:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527060495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527000466, max timestamp=1731527000502Num uniques 1018; 2024-11-13T19:43:20,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:43:20,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527000466, max timestamp=1731527000502Num uniques 1018; 2024-11-13T19:43:20,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:43:20,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:43:20,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741862_1038 (size=12151) 2024-11-13T19:43:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527060626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527000606, max timestamp=1731527000636Num uniques 1077; 2024-11-13T19:43:20,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:43:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527060624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527000606, max timestamp=1731527000636Num uniques 1077; 2024-11-13T19:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527000606, max timestamp=1731527000636Num uniques 1077; 2024-11-13T19:43:20,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:43:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:20,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731527000695, max timestamp=1731527000762Num uniques 1036; 2024-11-13T19:43:20,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-13T19:43:20,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731527000695, max timestamp=1731527000762Num uniques 1036; 2024-11-13T19:43:20,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-13T19:43:20,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731527000695, max timestamp=1731527000762Num uniques 1036; 2024-11-13T19:43:20,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-13T19:43:20,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527060867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:20,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:20,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527060872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527060875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527060879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527060878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:20,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3780312960934d27897c643908459186 2024-11-13T19:43:21,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527060989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/543cda9f746742cfb9518f9438317263 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263 2024-11-13T19:43:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527061017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527061017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527061026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527061027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:21,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:21,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:21,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T19:43:21,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/a54062f9a6494755971251dcebd6066e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e 2024-11-13T19:43:21,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T19:43:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3780312960934d27897c643908459186 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186 2024-11-13T19:43:21,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T19:43:21,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~115.73 KB/118512, currentSize=154.31 KB/158010 for 31328a0914442b3583375df8a5770e39 in 845ms, sequenceid=182, compaction requested=true 2024-11-13T19:43:21,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:21,129 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:43:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:21,129 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:43:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:21,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. because compaction request was cancelled 2024-11-13T19:43:21,131 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:21,133 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:21,134 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:21,134 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:21,134 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/77cabdb8330d4914b366b7f97f2033db, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=35.6 K 2024-11-13T19:43:21,135 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77cabdb8330d4914b366b7f97f2033db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731526998194 2024-11-13T19:43:21,136 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69305634d70d48b29aa87842b37993df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731526999880 2024-11-13T19:43:21,137 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3780312960934d27897c643908459186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731527000091 2024-11-13T19:43:21,158 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:21,159 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/97c8199c57734e5ab97687ba2ee0ce1f is 50, key is test_row_0/C:col10/1731527000091/Put/seqid=0 2024-11-13T19:43:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:21,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741863_1039 (size=12425) 2024-11-13T19:43:21,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:43:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:21,206 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=154.31 KB heapSize=423.47 KB 2024-11-13T19:43:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/343339dda8b44862ba9a5917376cbee6 is 50, key is test_row_0/A:col10/1731527000870/Put/seqid=0 2024-11-13T19:43:21,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:21,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741864_1040 (size=12301) 2024-11-13T19:43:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527061246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,269 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/343339dda8b44862ba9a5917376cbee6 2024-11-13T19:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527061245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/9071df797b7d49a29b70f407c14aad55 is 50, key is test_row_0/B:col10/1731527000870/Put/seqid=0 2024-11-13T19:43:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527061267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527061267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527061269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741865_1041 (size=12301) 2024-11-13T19:43:21,326 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/9071df797b7d49a29b70f407c14aad55 2024-11-13T19:43:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/c627751ba5bd494d9e77ef72a5c35dd3 is 50, key is test_row_0/C:col10/1731527000870/Put/seqid=0 2024-11-13T19:43:21,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741866_1042 (size=12301) 2024-11-13T19:43:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527061389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,395 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/c627751ba5bd494d9e77ef72a5c35dd3 2024-11-13T19:43:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527061405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527061406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527061406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/343339dda8b44862ba9a5917376cbee6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6 2024-11-13T19:43:21,436 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T19:43:21,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/9071df797b7d49a29b70f407c14aad55 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55 2024-11-13T19:43:21,450 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T19:43:21,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/c627751ba5bd494d9e77ef72a5c35dd3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3 2024-11-13T19:43:21,473 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T19:43:21,478 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~423.42 KB/433584, currentSize=40.25 KB/41220 for 31328a0914442b3583375df8a5770e39 in 269ms, sequenceid=272, compaction requested=true 2024-11-13T19:43:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T19:43:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T19:43:21,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T19:43:21,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-13T19:43:21,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4700 sec 2024-11-13T19:43:21,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:21,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:21,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:21,622 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/97c8199c57734e5ab97687ba2ee0ce1f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/97c8199c57734e5ab97687ba2ee0ce1f 2024-11-13T19:43:21,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdf46a5d5de14ffc84acd559e6641e16 is 50, key is test_row_0/A:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:21,644 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 97c8199c57734e5ab97687ba2ee0ce1f(size=12.1 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:21,644 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:21,644 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527001130; duration=0sec 2024-11-13T19:43:21,645 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:21,645 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:21,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741867_1043 (size=12301) 2024-11-13T19:43:21,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdf46a5d5de14ffc84acd559e6641e16 2024-11-13T19:43:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527001677, max timestamp=1731527001705Num uniques 859; 2024-11-13T19:43:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:43:21,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527001677, max timestamp=1731527001713Num uniques 859; 2024-11-13T19:43:21,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:43:21,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527001677, max timestamp=1731527001705Num uniques 859; 2024-11-13T19:43:21,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:43:21,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/fd126972e6fe4419a3dd36a5d94d4126 is 50, key is test_row_0/B:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741868_1044 (size=12301) 2024-11-13T19:43:21,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/fd126972e6fe4419a3dd36a5d94d4126 2024-11-13T19:43:21,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527061772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527061787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527061798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527061802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527061805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/bcad99a5773d4db9b0101c23ad6c01ff is 50, key is test_row_0/C:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:21,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741869_1045 (size=12301) 2024-11-13T19:43:21,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/bcad99a5773d4db9b0101c23ad6c01ff 2024-11-13T19:43:21,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdf46a5d5de14ffc84acd559e6641e16 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16 2024-11-13T19:43:21,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T19:43:21,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/fd126972e6fe4419a3dd36a5d94d4126 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126 2024-11-13T19:43:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527061925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527061925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527061926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527061926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527061927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:21,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T19:43:22,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/bcad99a5773d4db9b0101c23ad6c01ff as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff 2024-11-13T19:43:22,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T19:43:22,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 31328a0914442b3583375df8a5770e39 in 423ms, sequenceid=282, compaction requested=true 2024-11-13T19:43:22,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:22,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,023 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:22,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:22,046 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:22,046 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:22,046 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,047 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/210d471d9b29485593fa4194c94a8c12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=48.0 K 2024-11-13T19:43:22,047 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:22,047 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:22,047 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,047 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6c80cb22746a4f2bab9f0f40dc67ac83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=48.0 K 2024-11-13T19:43:22,048 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 210d471d9b29485593fa4194c94a8c12, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731526999880 2024-11-13T19:43:22,048 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c80cb22746a4f2bab9f0f40dc67ac83, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731526999880 2024-11-13T19:43:22,050 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a54062f9a6494755971251dcebd6066e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731527000091 2024-11-13T19:43:22,050 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543cda9f746742cfb9518f9438317263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731527000091 2024-11-13T19:43:22,050 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9071df797b7d49a29b70f407c14aad55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731527000852 2024-11-13T19:43:22,050 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343339dda8b44862ba9a5917376cbee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731527000852 2024-11-13T19:43:22,052 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting fd126972e6fe4419a3dd36a5d94d4126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:22,052 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf46a5d5de14ffc84acd559e6641e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:22,085 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:22,086 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/46eddb8ec2eb4930acb8ea6216c82371 is 50, key is test_row_0/B:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:22,098 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:22,099 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/772bbd8deb68499b9caafcc0a300404e is 50, key is test_row_0/A:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:22,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741870_1046 (size=12677) 2024-11-13T19:43:22,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-13T19:43:22,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:22,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741871_1047 (size=12677) 2024-11-13T19:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:43:22,197 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:22,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T19:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:22,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:22,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:22,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:22,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527062184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527062187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527062196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527062197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,231 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/772bbd8deb68499b9caafcc0a300404e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/772bbd8deb68499b9caafcc0a300404e 2024-11-13T19:43:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527062211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/6a36a87f343940b2ac6eacbd5cae527e is 50, key is test_row_0/A:col10/1731527001799/Put/seqid=0 2024-11-13T19:43:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,248 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 772bbd8deb68499b9caafcc0a300404e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:22,248 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,248 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=12, startTime=1731527002023; duration=0sec 2024-11-13T19:43:22,248 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:22,248 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:22,249 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:22,251 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:22,251 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:22,251 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,251 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/97c8199c57734e5ab97687ba2ee0ce1f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.2 K 2024-11-13T19:43:22,252 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97c8199c57734e5ab97687ba2ee0ce1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731527000091 2024-11-13T19:43:22,254 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c627751ba5bd494d9e77ef72a5c35dd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731527000852 2024-11-13T19:43:22,255 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcad99a5773d4db9b0101c23ad6c01ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:22,272 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:22,274 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/271a5f26b4cb4702bce97f00a5306049 is 50, key is test_row_0/C:col10/1731527001590/Put/seqid=0 2024-11-13T19:43:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741872_1048 (size=14741) 2024-11-13T19:43:22,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/6a36a87f343940b2ac6eacbd5cae527e 2024-11-13T19:43:22,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741873_1049 (size=12677) 2024-11-13T19:43:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,328 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/271a5f26b4cb4702bce97f00a5306049 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/271a5f26b4cb4702bce97f00a5306049 2024-11-13T19:43:22,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/8a735c83da68426ebc6e195269759892 is 50, key is test_row_0/B:col10/1731527001799/Put/seqid=0 2024-11-13T19:43:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527062317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527062316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,340 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 271a5f26b4cb4702bce97f00a5306049(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,344 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,345 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527002024; duration=0sec 2024-11-13T19:43:22,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,345 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,345 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:22,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527062334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741874_1050 (size=12301) 2024-11-13T19:43:22,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/8a735c83da68426ebc6e195269759892 2024-11-13T19:43:22,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527062344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527062344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cc7afefd60ca4e53adbbd30d007e6edd is 50, key is test_row_0/C:col10/1731527001799/Put/seqid=0 2024-11-13T19:43:22,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741875_1051 (size=12301) 2024-11-13T19:43:22,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cc7afefd60ca4e53adbbd30d007e6edd 2024-11-13T19:43:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/6a36a87f343940b2ac6eacbd5cae527e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e 2024-11-13T19:43:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e, entries=200, sequenceid=318, filesize=14.4 K 2024-11-13T19:43:22,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/8a735c83da68426ebc6e195269759892 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892 2024-11-13T19:43:22,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:43:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cc7afefd60ca4e53adbbd30d007e6edd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd 2024-11-13T19:43:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:22,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:43:22,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=53.67 KB/54960 for 31328a0914442b3583375df8a5770e39 in 382ms, sequenceid=318, compaction requested=false 2024-11-13T19:43:22,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:22,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:22,607 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/46eddb8ec2eb4930acb8ea6216c82371 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/46eddb8ec2eb4930acb8ea6216c82371 2024-11-13T19:43:22,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/3f697f2cd41249ab8888efcfc6b2e744 is 50, key is test_row_0/A:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:22,621 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 46eddb8ec2eb4930acb8ea6216c82371(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:22,621 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,621 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=12, startTime=1731527002023; duration=0sec 2024-11-13T19:43:22,622 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,622 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:22,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527062658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527062661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527062663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527062664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741876_1052 (size=12301) 2024-11-13T19:43:22,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/3f697f2cd41249ab8888efcfc6b2e744 2024-11-13T19:43:22,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527062666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aec518a8763943fca1e6830aff5b3802 is 50, key is test_row_0/B:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741877_1053 (size=12301) 2024-11-13T19:43:22,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aec518a8763943fca1e6830aff5b3802 2024-11-13T19:43:22,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1fe7e9df895f4fbb9d3fd48ae62edb1f is 50, key is test_row_0/C:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527062777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527062784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527062784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527062794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527062795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:22,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741878_1054 (size=12301) 2024-11-13T19:43:22,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1fe7e9df895f4fbb9d3fd48ae62edb1f 2024-11-13T19:43:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:22,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:22,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/3f697f2cd41249ab8888efcfc6b2e744 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744 2024-11-13T19:43:22,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:43:22,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aec518a8763943fca1e6830aff5b3802 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802 2024-11-13T19:43:22,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:43:22,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1fe7e9df895f4fbb9d3fd48ae62edb1f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f 2024-11-13T19:43:22,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:43:22,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 376ms, sequenceid=332, compaction requested=true 2024-11-13T19:43:22,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:22,932 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,933 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:22,946 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:22,946 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:22,946 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,947 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/46eddb8ec2eb4930acb8ea6216c82371, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.4 K 2024-11-13T19:43:22,947 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:22,947 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:22,948 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:22,948 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/772bbd8deb68499b9caafcc0a300404e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=38.8 K 2024-11-13T19:43:22,948 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 46eddb8ec2eb4930acb8ea6216c82371, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:22,949 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 772bbd8deb68499b9caafcc0a300404e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:22,949 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a735c83da68426ebc6e195269759892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731527001791 2024-11-13T19:43:22,950 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a36a87f343940b2ac6eacbd5cae527e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731527001697 2024-11-13T19:43:22,950 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f697f2cd41249ab8888efcfc6b2e744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:22,951 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting aec518a8763943fca1e6830aff5b3802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:22,993 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:22,994 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/220ca12e1cf84c529ae47934dcc0cc2e is 50, key is test_row_0/B:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:23,000 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:23,001 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1d24905f77234b43a55bd1796c98b74f is 50, key is test_row_0/A:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527002621, max timestamp=1731527002997Num uniques 859; 2024-11-13T19:43:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527002621, max timestamp=1731527003002Num uniques 859; 2024-11-13T19:43:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:23,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:23,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527002621, max timestamp=1731527003002Num uniques 859; 2024-11-13T19:43:23,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:23,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741879_1055 (size=12779) 2024-11-13T19:43:23,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/961d63fac60b4334a7a15dd6270f09bf is 50, key is test_row_0/A:col10/1731527002663/Put/seqid=0 2024-11-13T19:43:23,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527003049, max timestamp=1731527003082Num uniques 859; 2024-11-13T19:43:23,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527003049, max timestamp=1731527003082Num uniques 859; 2024-11-13T19:43:23,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:23,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527003049, max timestamp=1731527003082Num uniques 859; 2024-11-13T19:43:23,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:23,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:23,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741881_1057 (size=12301) 2024-11-13T19:43:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741880_1056 (size=12779) 2024-11-13T19:43:23,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527003107, max timestamp=1731527003158Num uniques 1118; 2024-11-13T19:43:23,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-13T19:43:23,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527003107, max timestamp=1731527003160Num uniques 1118; 2024-11-13T19:43:23,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-13T19:43:23,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527003107, max timestamp=1731527003158Num uniques 1118; 2024-11-13T19:43:23,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-13T19:43:23,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527003208, max timestamp=1731527003265Num uniques 1077; 2024-11-13T19:43:23,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-13T19:43:23,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527003208, max timestamp=1731527003269Num uniques 1077; 2024-11-13T19:43:23,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-13T19:43:23,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527003208, max timestamp=1731527003269Num uniques 1077; 2024-11-13T19:43:23,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-13T19:43:23,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527003324, max timestamp=1731527003330Num uniques 1086; 2024-11-13T19:43:23,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T19:43:23,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527003324, max timestamp=1731527003330Num uniques 1086; 2024-11-13T19:43:23,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-13T19:43:23,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527003324, max timestamp=1731527003330Num uniques 1086; 2024-11-13T19:43:23,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T19:43:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:23,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731527003386, max timestamp=1731527003412Num uniques 1045; 2024-11-13T19:43:23,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-13T19:43:23,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731527003386, max timestamp=1731527003412Num uniques 1009; 2024-11-13T19:43:23,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=15800, suffix heap size=132904,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=345 2024-11-13T19:43:23,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731527003386, max timestamp=1731527003412Num uniques 1045; 2024-11-13T19:43:23,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-13T19:43:23,441 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/220ca12e1cf84c529ae47934dcc0cc2e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/220ca12e1cf84c529ae47934dcc0cc2e 2024-11-13T19:43:23,453 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 220ca12e1cf84c529ae47934dcc0cc2e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:23,453 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:23,453 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527002933; duration=0sec 2024-11-13T19:43:23,453 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:23,453 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:23,454 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:23,456 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:23,456 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:23,456 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,457 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/271a5f26b4cb4702bce97f00a5306049, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.4 K 2024-11-13T19:43:23,462 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 271a5f26b4cb4702bce97f00a5306049, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731527001257 2024-11-13T19:43:23,463 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7afefd60ca4e53adbbd30d007e6edd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731527001791 2024-11-13T19:43:23,464 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe7e9df895f4fbb9d3fd48ae62edb1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:23,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/961d63fac60b4334a7a15dd6270f09bf 2024-11-13T19:43:23,506 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:23,507 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1c2577f4d3414c75a226ca2c5f3625ac is 50, key is test_row_0/C:col10/1731527002550/Put/seqid=0 2024-11-13T19:43:23,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,518 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1d24905f77234b43a55bd1796c98b74f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d24905f77234b43a55bd1796c98b74f 2024-11-13T19:43:23,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/7da5e6617add4db68c6d86435e5b683c is 50, key is test_row_0/B:col10/1731527002663/Put/seqid=0 2024-11-13T19:43:23,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741882_1058 (size=12779) 2024-11-13T19:43:23,561 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 1d24905f77234b43a55bd1796c98b74f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:23,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:23,561 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527002932; duration=0sec 2024-11-13T19:43:23,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:23,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:23,573 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1c2577f4d3414c75a226ca2c5f3625ac as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1c2577f4d3414c75a226ca2c5f3625ac 2024-11-13T19:43:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741883_1059 (size=12301) 2024-11-13T19:43:23,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/7da5e6617add4db68c6d86435e5b683c 2024-11-13T19:43:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527003519, max timestamp=1731527003599Num uniques 1204; 2024-11-13T19:43:23,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-13T19:43:23,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527003519, max timestamp=1731527003599Num uniques 1204; 2024-11-13T19:43:23,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527003519, max timestamp=1731527003599Num uniques 1204; 2024-11-13T19:43:23,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-13T19:43:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527063578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-13T19:43:23,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8dd3257c30d4451095682f036b77c862 is 50, key is test_row_0/C:col10/1731527002663/Put/seqid=0 2024-11-13T19:43:23,642 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 1c2577f4d3414c75a226ca2c5f3625ac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:23,642 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:23,643 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527002933; duration=0sec 2024-11-13T19:43:23,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:23,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:23,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741884_1060 (size=12301) 2024-11-13T19:43:23,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8dd3257c30d4451095682f036b77c862 2024-11-13T19:43:23,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/961d63fac60b4334a7a15dd6270f09bf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf 2024-11-13T19:43:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf, entries=150, sequenceid=355, filesize=12.0 K 2024-11-13T19:43:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/7da5e6617add4db68c6d86435e5b683c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c 2024-11-13T19:43:23,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=463, cellSize=21198, totalHeapSize=48392, min timestamp=1731527003624, max timestamp=1731527003690Num uniques 1013; 2024-11-13T19:43:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=21198, suffix heap size=133320,new segment heap size=48392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=463 2024-11-13T19:43:23,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c, entries=150, sequenceid=355, filesize=12.0 K 2024-11-13T19:43:23,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/8dd3257c30d4451095682f036b77c862 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862 2024-11-13T19:43:23,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=463, cellSize=21198, totalHeapSize=48392, min timestamp=1731527003624, max timestamp=1731527003690Num uniques 1013; 2024-11-13T19:43:23,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=21198, suffix heap size=133320,new segment heap size=48392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=463 2024-11-13T19:43:23,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=463, cellSize=21198, totalHeapSize=48392, min timestamp=1731527003624, max timestamp=1731527003690Num uniques 1013; 2024-11-13T19:43:23,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=21198, suffix heap size=133320,new segment heap size=48392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=463 2024-11-13T19:43:23,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862, entries=150, sequenceid=355, filesize=12.0 K 2024-11-13T19:43:23,739 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T19:43:23,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 736ms, sequenceid=355, compaction requested=false 2024-11-13T19:43:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:23,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=127.47 KB heapSize=336.56 KB 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:23,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:23,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/7b75e0f5509a41b8900e53191c09f46d is 50, key is test_row_0/A:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527063780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527063788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527063790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527063797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527063799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741885_1061 (size=12301) 2024-11-13T19:43:23,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/7b75e0f5509a41b8900e53191c09f46d 2024-11-13T19:43:23,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:23,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:23,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/167b7615d02f474fbe661a4e566ebee8 is 50, key is test_row_0/B:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:23,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527063915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527063915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527063915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527063934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527063934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:23,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741886_1062 (size=12301) 2024-11-13T19:43:24,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:24,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527064124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527064124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527064124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527064154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527064157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:24,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:24,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/167b7615d02f474fbe661a4e566ebee8 2024-11-13T19:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:24,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b493f516e9014e488ef23100641d58c2 is 50, key is test_row_0/C:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527064434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527064434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741887_1063 (size=12301) 2024-11-13T19:43:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527064434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b493f516e9014e488ef23100641d58c2 2024-11-13T19:43:24,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527064475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527064475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:24,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/7b75e0f5509a41b8900e53191c09f46d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d 2024-11-13T19:43:24,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T19:43:24,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/167b7615d02f474fbe661a4e566ebee8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8 2024-11-13T19:43:24,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T19:43:24,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b493f516e9014e488ef23100641d58c2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2 2024-11-13T19:43:24,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T19:43:24,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~336.52 KB/344592, currentSize=60.38 KB/61830 for 31328a0914442b3583375df8a5770e39 in 775ms, sequenceid=491, compaction requested=true 2024-11-13T19:43:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:24,526 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:24,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:24,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:24,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:24,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:24,528 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,528 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d24905f77234b43a55bd1796c98b74f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.5 K 2024-11-13T19:43:24,529 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d24905f77234b43a55bd1796c98b74f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:24,529 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 961d63fac60b4334a7a15dd6270f09bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527002663 2024-11-13T19:43:24,530 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b75e0f5509a41b8900e53191c09f46d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:24,531 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:24,533 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,534 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/220ca12e1cf84c529ae47934dcc0cc2e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.5 K 2024-11-13T19:43:24,535 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 220ca12e1cf84c529ae47934dcc0cc2e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:24,536 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da5e6617add4db68c6d86435e5b683c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527002663 2024-11-13T19:43:24,537 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 167b7615d02f474fbe661a4e566ebee8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:24,557 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:24,558 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/4f4e95148f0043a4a12d29e29ad4c4b3 is 50, key is test_row_0/A:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:24,568 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:24,569 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/d04a28f7fd91432ebe89e4b13940cd51 is 50, key is test_row_0/B:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:24,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741888_1064 (size=12881) 2024-11-13T19:43:24,611 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/4f4e95148f0043a4a12d29e29ad4c4b3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4f4e95148f0043a4a12d29e29ad4c4b3 2024-11-13T19:43:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741889_1065 (size=12881) 2024-11-13T19:43:24,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:43:24,636 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 4f4e95148f0043a4a12d29e29ad4c4b3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:24,636 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:24,636 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527004526; duration=0sec 2024-11-13T19:43:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,636 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:24,637 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:24,637 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:43:24,637 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:24,640 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:24,640 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:24,640 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:24,641 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1c2577f4d3414c75a226ca2c5f3625ac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.5 K 2024-11-13T19:43:24,643 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2577f4d3414c75a226ca2c5f3625ac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731527002184 2024-11-13T19:43:24,650 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dd3257c30d4451095682f036b77c862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527002663 2024-11-13T19:43:24,651 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b493f516e9014e488ef23100641d58c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:24,652 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/d04a28f7fd91432ebe89e4b13940cd51 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d04a28f7fd91432ebe89e4b13940cd51 2024-11-13T19:43:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f2e0a2e6d586484dbde5bbf147006995 is 50, key is test_row_0/A:col10/1731527003772/Put/seqid=0 2024-11-13T19:43:24,667 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into d04a28f7fd91432ebe89e4b13940cd51(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:24,667 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:24,667 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527004531; duration=0sec 2024-11-13T19:43:24,667 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:24,667 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:24,696 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:24,697 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ddd364a19bd14fe2b557565f74989c7f is 50, key is test_row_0/C:col10/1731527003730/Put/seqid=0 2024-11-13T19:43:24,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741890_1066 (size=12301) 2024-11-13T19:43:24,718 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f2e0a2e6d586484dbde5bbf147006995 2024-11-13T19:43:24,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741891_1067 (size=12881) 2024-11-13T19:43:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/ee8e0d28e12b42aca944cfced1a026e6 is 50, key is test_row_0/B:col10/1731527003772/Put/seqid=0 2024-11-13T19:43:24,794 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ddd364a19bd14fe2b557565f74989c7f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ddd364a19bd14fe2b557565f74989c7f 2024-11-13T19:43:24,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741892_1068 (size=12301) 2024-11-13T19:43:24,813 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/ee8e0d28e12b42aca944cfced1a026e6 2024-11-13T19:43:24,826 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into ddd364a19bd14fe2b557565f74989c7f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:24,826 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:24,826 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527004532; duration=0sec 2024-11-13T19:43:24,826 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:24,826 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3bff4bebf7b8459a8983fa804d0a1bb5 is 50, key is test_row_0/C:col10/1731527003772/Put/seqid=0 2024-11-13T19:43:24,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741893_1069 (size=12301) 2024-11-13T19:43:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:24,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527065049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731527004988, max timestamp=1731527005076Num uniques 859; 2024-11-13T19:43:25,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T19:43:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731527004988, max timestamp=1731527005076Num uniques 859; 2024-11-13T19:43:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-13T19:43:25,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527005039, max timestamp=1731527005076Num uniques 859; 2024-11-13T19:43:25,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527065174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527065176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527065191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527065193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527065195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,305 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3bff4bebf7b8459a8983fa804d0a1bb5 2024-11-13T19:43:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f2e0a2e6d586484dbde5bbf147006995 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995 2024-11-13T19:43:25,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527065303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527065314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527065317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527065323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,351 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T19:43:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/ee8e0d28e12b42aca944cfced1a026e6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6 2024-11-13T19:43:25,364 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T19:43:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/3bff4bebf7b8459a8983fa804d0a1bb5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5 2024-11-13T19:43:25,380 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T19:43:25,399 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 31328a0914442b3583375df8a5770e39 in 762ms, sequenceid=504, compaction requested=false 2024-11-13T19:43:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T19:43:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T19:43:25,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T19:43:25,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2010 sec 2024-11-13T19:43:25,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.2100 sec 2024-11-13T19:43:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:25,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=129.71 KB heapSize=356.08 KB 2024-11-13T19:43:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:25,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:25,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:25,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:25,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/edd2cbba0e78482998ba332b335e2857 is 50, key is test_row_0/A:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741894_1070 (size=14741) 2024-11-13T19:43:25,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/edd2cbba0e78482998ba332b335e2857 2024-11-13T19:43:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527065545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e6f9060f439744259d42584bc06c9f48 is 50, key is test_row_0/B:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527065555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527065559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527065561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527065563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741895_1071 (size=12301) 2024-11-13T19:43:25,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e6f9060f439744259d42584bc06c9f48 2024-11-13T19:43:25,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/0410f690b67c4a44877fabbd8d52a8ba is 50, key is test_row_0/C:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:25,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527065678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527065704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527065704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527065705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741896_1072 (size=12301) 2024-11-13T19:43:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527065709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:25,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/0410f690b67c4a44877fabbd8d52a8ba 2024-11-13T19:43:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:25,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:25,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/edd2cbba0e78482998ba332b335e2857 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857 2024-11-13T19:43:25,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857, entries=200, sequenceid=539, filesize=14.4 K 2024-11-13T19:43:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/e6f9060f439744259d42584bc06c9f48 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48 2024-11-13T19:43:25,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48, entries=150, sequenceid=539, filesize=12.0 K 2024-11-13T19:43:25,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/0410f690b67c4a44877fabbd8d52a8ba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba 2024-11-13T19:43:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba, entries=150, sequenceid=539, filesize=12.0 K 2024-11-13T19:43:25,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~356.03 KB/364576, currentSize=60.38 KB/61830 for 31328a0914442b3583375df8a5770e39 in 385ms, sequenceid=539, compaction requested=true 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:25,832 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:25,833 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:25,837 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:25,838 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:25,838 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:25,838 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4f4e95148f0043a4a12d29e29ad4c4b3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=39.0 K 2024-11-13T19:43:25,839 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:25,839 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:25,839 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:25,839 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d04a28f7fd91432ebe89e4b13940cd51, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.6 K 2024-11-13T19:43:25,840 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f4e95148f0043a4a12d29e29ad4c4b3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:25,840 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d04a28f7fd91432ebe89e4b13940cd51, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:25,840 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e0a2e6d586484dbde5bbf147006995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527003772 2024-11-13T19:43:25,842 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8e0d28e12b42aca944cfced1a026e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527003772 2024-11-13T19:43:25,844 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd2cbba0e78482998ba332b335e2857, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005159 2024-11-13T19:43:25,844 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f9060f439744259d42584bc06c9f48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005173 2024-11-13T19:43:25,880 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#60 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:25,881 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/ef04e947a3a9454a9f9eaa5be7cf2a8a is 50, key is test_row_0/B:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:25,903 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#61 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:25,904 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/d1e2a5e6c0544f8f9ef61d338111f741 is 50, key is test_row_0/A:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:25,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:25,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741897_1073 (size=12983) 2024-11-13T19:43:25,947 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/ef04e947a3a9454a9f9eaa5be7cf2a8a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ef04e947a3a9454a9f9eaa5be7cf2a8a 2024-11-13T19:43:25,959 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into ef04e947a3a9454a9f9eaa5be7cf2a8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:25,959 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:25,959 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527005832; duration=0sec 2024-11-13T19:43:25,959 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:25,959 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:25,959 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:25,961 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:25,962 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:25,962 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:25,962 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ddd364a19bd14fe2b557565f74989c7f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.6 K 2024-11-13T19:43:25,964 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd364a19bd14fe2b557565f74989c7f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731527003730 2024-11-13T19:43:25,965 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bff4bebf7b8459a8983fa804d0a1bb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527003772 2024-11-13T19:43:25,965 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0410f690b67c4a44877fabbd8d52a8ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005173 2024-11-13T19:43:25,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1d591f3a78184c79b4ec20ec42dc8744 is 50, key is test_row_0/A:col10/1731527005538/Put/seqid=0 2024-11-13T19:43:26,005 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:26,006 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/0b38e0f8a5b34b4dad143d723c08736b is 50, key is test_row_0/C:col10/1731527005173/Put/seqid=0 2024-11-13T19:43:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741898_1074 (size=12983) 2024-11-13T19:43:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741899_1075 (size=14741) 2024-11-13T19:43:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527066038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741900_1076 (size=12983) 2024-11-13T19:43:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527066040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527066046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527066049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527066050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527066154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527066175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527066175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527066175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527066176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527066374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527066383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:43:26,395 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:26,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T19:43:26,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:26,399 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:26,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527066384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527066384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527066394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:26,417 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/d1e2a5e6c0544f8f9ef61d338111f741 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/d1e2a5e6c0544f8f9ef61d338111f741 2024-11-13T19:43:26,424 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into d1e2a5e6c0544f8f9ef61d338111f741(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:26,425 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:26,425 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527005832; duration=0sec 2024-11-13T19:43:26,425 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:26,425 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:26,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1d591f3a78184c79b4ec20ec42dc8744 2024-11-13T19:43:26,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cc0c8b7a38de4fabb0fbca7466f706bd is 50, key is test_row_0/B:col10/1731527005538/Put/seqid=0 2024-11-13T19:43:26,466 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/0b38e0f8a5b34b4dad143d723c08736b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0b38e0f8a5b34b4dad143d723c08736b 2024-11-13T19:43:26,474 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 0b38e0f8a5b34b4dad143d723c08736b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:26,474 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:26,474 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527005832; duration=0sec 2024-11-13T19:43:26,474 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:26,475 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:26,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741901_1077 (size=12301) 2024-11-13T19:43:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:26,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cc0c8b7a38de4fabb0fbca7466f706bd 2024-11-13T19:43:26,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/16f86515b5394c96bc75330d636795eb is 50, key is test_row_0/C:col10/1731527005538/Put/seqid=0 2024-11-13T19:43:26,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741902_1078 (size=12301) 2024-11-13T19:43:26,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/16f86515b5394c96bc75330d636795eb 2024-11-13T19:43:26,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/1d591f3a78184c79b4ec20ec42dc8744 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744 2024-11-13T19:43:26,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:26,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:26,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744, entries=200, sequenceid=552, filesize=14.4 K 2024-11-13T19:43:26,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/cc0c8b7a38de4fabb0fbca7466f706bd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd 2024-11-13T19:43:26,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd, entries=150, sequenceid=552, filesize=12.0 K 2024-11-13T19:43:26,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/16f86515b5394c96bc75330d636795eb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb 2024-11-13T19:43:26,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb, entries=150, sequenceid=552, filesize=12.0 K 2024-11-13T19:43:26,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 694ms, sequenceid=552, compaction requested=false 2024-11-13T19:43:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:26,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527006035, max timestamp=1731527006685Num uniques 859; 2024-11-13T19:43:26,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:26,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527006035, max timestamp=1731527006685Num uniques 859; 2024-11-13T19:43:26,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:26,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527006035, max timestamp=1731527006685Num uniques 859; 2024-11-13T19:43:26,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:26,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:26,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:26,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/229af55a792c499ca1f5d48e4c3b05a6 is 50, key is test_row_0/A:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741903_1079 (size=12301) 2024-11-13T19:43:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:26,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527006720, max timestamp=1731527006739Num uniques 859; 2024-11-13T19:43:26,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:26,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527006720, max timestamp=1731527006739Num uniques 859; 2024-11-13T19:43:26,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:26,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527006720, max timestamp=1731527006739Num uniques 859; 2024-11-13T19:43:26,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:26,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527006790, max timestamp=1731527006815Num uniques 1068; 2024-11-13T19:43:26,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:26,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527006790, max timestamp=1731527006815Num uniques 1068; 2024-11-13T19:43:26,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:26,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527006790, max timestamp=1731527006815Num uniques 1068; 2024-11-13T19:43:26,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T19:43:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527006852, max timestamp=1731527006886Num uniques 1127; 2024-11-13T19:43:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T19:43:26,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527006852, max timestamp=1731527006886Num uniques 1127; 2024-11-13T19:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527006852, max timestamp=1731527006886Num uniques 1127; 2024-11-13T19:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T19:43:26,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T19:43:26,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:26,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:26,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731527006897, max timestamp=1731527006944Num uniques 1086; 2024-11-13T19:43:26,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T19:43:26,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731527006897, max timestamp=1731527006944Num uniques 1086; 2024-11-13T19:43:26,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T19:43:26,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527006897, max timestamp=1731527006944Num uniques 1086; 2024-11-13T19:43:26,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-13T19:43:27,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527006984, max timestamp=1731527007009Num uniques 1095; 2024-11-13T19:43:27,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527006984, max timestamp=1731527007009Num uniques 1095; 2024-11-13T19:43:27,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T19:43:27,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T19:43:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527006984, max timestamp=1731527007010Num uniques 1009; 2024-11-13T19:43:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-11-13T19:43:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:27,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:27,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527007064, max timestamp=1731527007081Num uniques 1054; 2024-11-13T19:43:27,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:43:27,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527007064, max timestamp=1731527007081Num uniques 1054; 2024-11-13T19:43:27,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:43:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527007064, max timestamp=1731527007081Num uniques 1054; 2024-11-13T19:43:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:43:27,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/229af55a792c499ca1f5d48e4c3b05a6 2024-11-13T19:43:27,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/1880e3b7e31d45e6802f52cefff0f7f7 is 50, key is test_row_0/B:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:27,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741904_1080 (size=12301) 2024-11-13T19:43:27,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731527007158, max timestamp=1731527007190Num uniques 1013; 2024-11-13T19:43:27,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731527007158, max timestamp=1731527007190Num uniques 1013; 2024-11-13T19:43:27,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=14328, suffix heap size=133320,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=313 2024-11-13T19:43:27,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=14328, suffix heap size=133320,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=313 2024-11-13T19:43:27,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731527007158, max timestamp=1731527007190Num uniques 1013; 2024-11-13T19:43:27,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=14328, suffix heap size=133320,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=313 2024-11-13T19:43:27,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731527007215, max timestamp=1731527007263Num uniques 1172; 2024-11-13T19:43:27,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-11-13T19:43:27,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527007241, max timestamp=1731527007263Num uniques 1172; 2024-11-13T19:43:27,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=6870, suffix heap size=149856,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=150 2024-11-13T19:43:27,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527007241, max timestamp=1731527007263Num uniques 1172; 2024-11-13T19:43:27,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=6870, suffix heap size=149856,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=150 2024-11-13T19:43:27,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8285, totalHeapSize=19064, min timestamp=1731527007299, max timestamp=1731527007328Num uniques 1181; 2024-11-13T19:43:27,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54085, new segment data size=8285, suffix heap size=150792,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=181 2024-11-13T19:43:27,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731527007299, max timestamp=1731527007328Num uniques 1009; 2024-11-13T19:43:27,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8286, suffix heap size=132904,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=181 2024-11-13T19:43:27,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731527007299, max timestamp=1731527007328Num uniques 1009; 2024-11-13T19:43:27,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8286, suffix heap size=132904,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=181 2024-11-13T19:43:27,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527007398, max timestamp=1731527007420Num uniques 1040; 2024-11-13T19:43:27,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527007398, max timestamp=1731527007420Num uniques 1040; 2024-11-13T19:43:27,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-13T19:43:27,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-13T19:43:27,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527007398, max timestamp=1731527007420Num uniques 1040; 2024-11-13T19:43:27,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-13T19:43:27,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527007486, max timestamp=1731527007498Num uniques 1009; 2024-11-13T19:43:27,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527007486, max timestamp=1731527007498Num uniques 1009; 2024-11-13T19:43:27,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9114, suffix heap size=132904,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=199 2024-11-13T19:43:27,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9114, suffix heap size=132904,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=199 2024-11-13T19:43:27,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527007486, max timestamp=1731527007498Num uniques 1009; 2024-11-13T19:43:27,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9114, suffix heap size=132904,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=199 2024-11-13T19:43:27,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/1880e3b7e31d45e6802f52cefff0f7f7 2024-11-13T19:43:27,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=558, cellSize=25548, totalHeapSize=58272, min timestamp=1731527007530, max timestamp=1731527007582Num uniques 1058; 2024-11-13T19:43:27,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=25548, suffix heap size=138000,new segment heap size=58272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=558 2024-11-13T19:43:27,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=558, cellSize=25550, totalHeapSize=58272, min timestamp=1731527007530, max timestamp=1731527007582Num uniques 1058; 2024-11-13T19:43:27,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=25550, suffix heap size=138000,new segment heap size=58272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=558 2024-11-13T19:43:27,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6964908d1c944060b6185d33fd9456c5 is 50, key is test_row_0/C:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:27,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=558, cellSize=25550, totalHeapSize=58272, min timestamp=1731527007530, max timestamp=1731527007582Num uniques 1058; 2024-11-13T19:43:27,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=25550, suffix heap size=138000,new segment heap size=58272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=558 2024-11-13T19:43:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527067638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527067647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527067653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527067658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527067667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741905_1081 (size=12301) 2024-11-13T19:43:27,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6964908d1c944060b6185d33fd9456c5 2024-11-13T19:43:27,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:27,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:27,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/229af55a792c499ca1f5d48e4c3b05a6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6 2024-11-13T19:43:27,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6, entries=150, sequenceid=576, filesize=12.0 K 2024-11-13T19:43:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/1880e3b7e31d45e6802f52cefff0f7f7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7 2024-11-13T19:43:27,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7, entries=150, sequenceid=576, filesize=12.0 K 2024-11-13T19:43:27,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6964908d1c944060b6185d33fd9456c5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5 2024-11-13T19:43:27,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527067781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527067781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527067784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527067784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527067784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:27,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5, entries=150, sequenceid=576, filesize=12.0 K 2024-11-13T19:43:27,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=174.43 KB/178620 for 31328a0914442b3583375df8a5770e39 in 1108ms, sequenceid=576, compaction requested=true 2024-11-13T19:43:27,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:27,798 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:27,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:27,799 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:27,806 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:27,807 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:27,807 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,807 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:27,807 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:27,807 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,807 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ef04e947a3a9454a9f9eaa5be7cf2a8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.7 K 2024-11-13T19:43:27,808 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/d1e2a5e6c0544f8f9ef61d338111f741, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=39.1 K 2024-11-13T19:43:27,808 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ef04e947a3a9454a9f9eaa5be7cf2a8a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005173 2024-11-13T19:43:27,808 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e2a5e6c0544f8f9ef61d338111f741, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005173 2024-11-13T19:43:27,809 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0c8b7a38de4fabb0fbca7466f706bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731527005538 2024-11-13T19:43:27,809 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d591f3a78184c79b4ec20ec42dc8744, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731527005538 2024-11-13T19:43:27,809 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1880e3b7e31d45e6802f52cefff0f7f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:27,810 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 229af55a792c499ca1f5d48e4c3b05a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:27,838 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:27,839 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/68a2cc830d3646398de3101fd810ba1d is 50, key is test_row_0/B:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:27,839 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:27,840 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/89cc145aed214167bbdb1fcf4116f5fd is 50, key is test_row_0/A:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:27,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:43:27,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,897 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=174.43 KB heapSize=467.11 KB 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:27,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741906_1082 (size=13085) 2024-11-13T19:43:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10a82f540a584fd0ab4fe5792209abc3 is 50, key is test_row_0/A:col10/1731527007651/Put/seqid=0 2024-11-13T19:43:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741907_1083 (size=13085) 2024-11-13T19:43:27,954 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/68a2cc830d3646398de3101fd810ba1d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/68a2cc830d3646398de3101fd810ba1d 2024-11-13T19:43:27,976 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 68a2cc830d3646398de3101fd810ba1d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:27,976 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:27,976 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527007799; duration=0sec 2024-11-13T19:43:27,976 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:27,976 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:27,977 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:27,980 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:27,980 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/89cc145aed214167bbdb1fcf4116f5fd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/89cc145aed214167bbdb1fcf4116f5fd 2024-11-13T19:43:27,980 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:27,980 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:27,980 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0b38e0f8a5b34b4dad143d723c08736b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.7 K 2024-11-13T19:43:27,981 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b38e0f8a5b34b4dad143d723c08736b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731527005173 2024-11-13T19:43:27,982 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f86515b5394c96bc75330d636795eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731527005538 2024-11-13T19:43:27,983 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6964908d1c944060b6185d33fd9456c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:27,997 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 89cc145aed214167bbdb1fcf4116f5fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:27,997 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:27,997 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527007798; duration=0sec 2024-11-13T19:43:27,997 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:27,997 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527068004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741908_1084 (size=12301) 2024-11-13T19:43:28,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527068008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,032 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527068010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527068011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,033 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/f8175fabe43742d589de7ba36cf54064 is 50, key is test_row_0/C:col10/1731527006035/Put/seqid=0 2024-11-13T19:43:28,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527068012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,034 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10a82f540a584fd0ab4fe5792209abc3 2024-11-13T19:43:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/91fcc59ce14f47ceaf627cfc31a7651a is 50, key is test_row_0/B:col10/1731527007651/Put/seqid=0 2024-11-13T19:43:28,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741909_1085 (size=13085) 2024-11-13T19:43:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741910_1086 (size=12301) 2024-11-13T19:43:28,117 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/91fcc59ce14f47ceaf627cfc31a7651a 2024-11-13T19:43:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/a0819b41340a42a084e11860ac073b48 is 50, key is test_row_0/C:col10/1731527007651/Put/seqid=0 2024-11-13T19:43:28,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527068144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527068145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527068145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527068145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741911_1087 (size=12301) 2024-11-13T19:43:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527068325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527068353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527068364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527068364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527068365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/f8175fabe43742d589de7ba36cf54064 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/f8175fabe43742d589de7ba36cf54064 2024-11-13T19:43:28,546 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into f8175fabe43742d589de7ba36cf54064(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:28,546 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:28,546 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527007799; duration=0sec 2024-11-13T19:43:28,546 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:28,546 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:28,562 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/a0819b41340a42a084e11860ac073b48 2024-11-13T19:43:28,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/10a82f540a584fd0ab4fe5792209abc3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3 2024-11-13T19:43:28,615 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3, entries=150, sequenceid=800, filesize=12.0 K 2024-11-13T19:43:28,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/91fcc59ce14f47ceaf627cfc31a7651a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a 2024-11-13T19:43:28,646 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a, entries=150, sequenceid=800, filesize=12.0 K 2024-11-13T19:43:28,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/a0819b41340a42a084e11860ac073b48 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48 2024-11-13T19:43:28,658 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48, entries=150, sequenceid=800, filesize=12.0 K 2024-11-13T19:43:28,660 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~467.06 KB/478272, currentSize=26.84 KB/27480 for 31328a0914442b3583375df8a5770e39 in 786ms, sequenceid=800, compaction requested=false 2024-11-13T19:43:28,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:28,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:28,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T19:43:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T19:43:28,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T19:43:28,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2770 sec 2024-11-13T19:43:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:28,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:28,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.2910 sec 2024-11-13T19:43:28,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/828b70c3ee214fd68907c1dd00d60289 is 50, key is test_row_0/A:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527068759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527008734, max timestamp=1731527008779Num uniques 859; 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527008734, max timestamp=1731527008779Num uniques 859; 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527008734, max timestamp=1731527008779Num uniques 859; 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:28,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:28,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741912_1088 (size=22065) 2024-11-13T19:43:28,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/828b70c3ee214fd68907c1dd00d60289 2024-11-13T19:43:28,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/96d6b46b7eda4f2d83c3992eb7fc594b is 50, key is test_row_0/B:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:28,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741913_1089 (size=12301) 2024-11-13T19:43:28,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/96d6b46b7eda4f2d83c3992eb7fc594b 2024-11-13T19:43:28,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527068850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527068852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527068865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527068863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b7d94d86bf214399b5202ce9e349b7a6 is 50, key is test_row_0/C:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527068893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741914_1090 (size=12301) 2024-11-13T19:43:28,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=814 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b7d94d86bf214399b5202ce9e349b7a6 2024-11-13T19:43:28,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/828b70c3ee214fd68907c1dd00d60289 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289 2024-11-13T19:43:28,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289, entries=350, sequenceid=814, filesize=21.5 K 2024-11-13T19:43:28,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/96d6b46b7eda4f2d83c3992eb7fc594b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b 2024-11-13T19:43:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527068973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527068973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b, entries=150, sequenceid=814, filesize=12.0 K 2024-11-13T19:43:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527068987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/b7d94d86bf214399b5202ce9e349b7a6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6 2024-11-13T19:43:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527068994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6, entries=150, sequenceid=814, filesize=12.0 K 2024-11-13T19:43:29,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 31328a0914442b3583375df8a5770e39 in 323ms, sequenceid=814, compaction requested=true 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:29,010 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:29,010 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:29,011 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:29,012 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:29,012 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:29,012 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/89cc145aed214167bbdb1fcf4116f5fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=46.3 K 2024-11-13T19:43:29,012 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:29,013 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:29,013 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cc145aed214167bbdb1fcf4116f5fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:29,013 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:29,013 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/68a2cc830d3646398de3101fd810ba1d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.8 K 2024-11-13T19:43:29,013 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a82f540a584fd0ab4fe5792209abc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1731527007634 2024-11-13T19:43:29,014 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a2cc830d3646398de3101fd810ba1d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:29,014 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828b70c3ee214fd68907c1dd00d60289, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008004 2024-11-13T19:43:29,014 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 91fcc59ce14f47ceaf627cfc31a7651a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1731527007634 2024-11-13T19:43:29,015 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d6b46b7eda4f2d83c3992eb7fc594b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008680 2024-11-13T19:43:29,058 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:29,059 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/eec02e5e1eb54e689e1cd886b899cfbf is 50, key is test_row_0/A:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:29,063 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:29,064 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/3f8da903304a4ffab295f231adb47ac1 is 50, key is test_row_0/B:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:29,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:29,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741916_1092 (size=13187) 2024-11-13T19:43:29,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741915_1091 (size=13187) 2024-11-13T19:43:29,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f920c3f9ebf3461fb7c930f3bc3e3468 is 50, key is test_row_0/A:col10/1731527008847/Put/seqid=0 2024-11-13T19:43:29,174 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/eec02e5e1eb54e689e1cd886b899cfbf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eec02e5e1eb54e689e1cd886b899cfbf 2024-11-13T19:43:29,178 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/3f8da903304a4ffab295f231adb47ac1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3f8da903304a4ffab295f231adb47ac1 2024-11-13T19:43:29,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741917_1093 (size=14741) 2024-11-13T19:43:29,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f920c3f9ebf3461fb7c930f3bc3e3468 2024-11-13T19:43:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527069204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,226 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into eec02e5e1eb54e689e1cd886b899cfbf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:29,226 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 3f8da903304a4ffab295f231adb47ac1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:29,226 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:29,226 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527009010; duration=0sec 2024-11-13T19:43:29,226 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:29,226 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:29,226 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:29,227 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:29,227 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527009010; duration=0sec 2024-11-13T19:43:29,228 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:29,228 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527069206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,236 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:29,236 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:29,236 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:29,236 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/f8175fabe43742d589de7ba36cf54064, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.8 K 2024-11-13T19:43:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527069210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,242 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f8175fabe43742d589de7ba36cf54064, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731527006035 2024-11-13T19:43:29,243 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a0819b41340a42a084e11860ac073b48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1731527007634 2024-11-13T19:43:29,244 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d94d86bf214399b5202ce9e349b7a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008680 2024-11-13T19:43:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527069217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527069213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/5073e973a7554fb7b3ca55e14d98decb is 50, key is test_row_0/B:col10/1731527008847/Put/seqid=0 2024-11-13T19:43:29,264 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:29,265 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6c567b9823c64006869d468035a4942e is 50, key is test_row_0/C:col10/1731527008685/Put/seqid=0 2024-11-13T19:43:29,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741918_1094 (size=12301) 2024-11-13T19:43:29,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/5073e973a7554fb7b3ca55e14d98decb 2024-11-13T19:43:29,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6e671d905bea42e488b8b5201210921d is 50, key is test_row_0/C:col10/1731527008847/Put/seqid=0 2024-11-13T19:43:29,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741919_1095 (size=13187) 2024-11-13T19:43:29,350 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6c567b9823c64006869d468035a4942e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6c567b9823c64006869d468035a4942e 2024-11-13T19:43:29,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527069344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527069344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,363 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 6c567b9823c64006869d468035a4942e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:29,363 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:29,363 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527009010; duration=0sec 2024-11-13T19:43:29,364 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:29,364 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527069355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741920_1096 (size=12301) 2024-11-13T19:43:29,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6e671d905bea42e488b8b5201210921d 2024-11-13T19:43:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527069369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/f920c3f9ebf3461fb7c930f3bc3e3468 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468 2024-11-13T19:43:29,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468, entries=200, sequenceid=849, filesize=14.4 K 2024-11-13T19:43:29,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/5073e973a7554fb7b3ca55e14d98decb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb 2024-11-13T19:43:29,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb, entries=150, sequenceid=849, filesize=12.0 K 2024-11-13T19:43:29,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/6e671d905bea42e488b8b5201210921d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d 2024-11-13T19:43:29,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d, entries=150, sequenceid=849, filesize=12.0 K 2024-11-13T19:43:29,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 31328a0914442b3583375df8a5770e39 in 327ms, sequenceid=849, compaction requested=false 2024-11-13T19:43:29,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:29,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:29,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:29,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/2af47da124af4505a8eeb52fc4f3dfd2 is 50, key is test_row_0/A:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:29,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741921_1097 (size=14741) 2024-11-13T19:43:29,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/2af47da124af4505a8eeb52fc4f3dfd2 2024-11-13T19:43:29,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527069605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/927e0845e62a4ba3aa9f8eb46731f9f3 is 50, key is test_row_0/B:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527069606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527069609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527069610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527069610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741922_1098 (size=12301) 2024-11-13T19:43:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527069726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527069736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527069736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527069737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527069924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527069944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527069945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527069946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527069956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/927e0845e62a4ba3aa9f8eb46731f9f3 2024-11-13T19:43:30,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/288f5a35643b490e8972113f133d96aa is 50, key is test_row_0/C:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741923_1099 (size=12301) 2024-11-13T19:43:30,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527070253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527070264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527070264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527070264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527070433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/288f5a35643b490e8972113f133d96aa 2024-11-13T19:43:30,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/2af47da124af4505a8eeb52fc4f3dfd2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2 2024-11-13T19:43:30,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2, entries=200, sequenceid=864, filesize=14.4 K 2024-11-13T19:43:30,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/927e0845e62a4ba3aa9f8eb46731f9f3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3 2024-11-13T19:43:30,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3, entries=150, sequenceid=864, filesize=12.0 K 2024-11-13T19:43:30,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/288f5a35643b490e8972113f133d96aa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa 2024-11-13T19:43:30,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa, entries=150, sequenceid=864, filesize=12.0 K 2024-11-13T19:43:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:43:30,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 1033ms, sequenceid=864, compaction requested=true 2024-11-13T19:43:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:30,566 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:30,566 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:30,566 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:30,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:30,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:30,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:30,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:30,568 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:30,568 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:30,569 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:30,569 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eec02e5e1eb54e689e1cd886b899cfbf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=41.7 K 2024-11-13T19:43:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T19:43:30,569 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec02e5e1eb54e689e1cd886b899cfbf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008680 2024-11-13T19:43:30,570 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:30,570 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:30,570 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:30,570 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3f8da903304a4ffab295f231adb47ac1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.9 K 2024-11-13T19:43:30,571 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f920c3f9ebf3461fb7c930f3bc3e3468, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1731527008845 2024-11-13T19:43:30,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:30,572 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f8da903304a4ffab295f231adb47ac1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008680 2024-11-13T19:43:30,573 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af47da124af4505a8eeb52fc4f3dfd2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:30,573 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5073e973a7554fb7b3ca55e14d98decb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1731527008845 2024-11-13T19:43:30,574 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 927e0845e62a4ba3aa9f8eb46731f9f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:30,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:30,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:30,588 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#87 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:30,589 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/93e9785644804a90a95cc6e247e80ffa is 50, key is test_row_0/A:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:30,597 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:30,598 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/d65aa149fc5f430ea34324e8afb1a1c4 is 50, key is test_row_0/B:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741924_1100 (size=13289) 2024-11-13T19:43:30,636 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/93e9785644804a90a95cc6e247e80ffa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/93e9785644804a90a95cc6e247e80ffa 2024-11-13T19:43:30,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741925_1101 (size=13289) 2024-11-13T19:43:30,647 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into 93e9785644804a90a95cc6e247e80ffa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:30,647 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:30,647 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527010566; duration=0sec 2024-11-13T19:43:30,647 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:30,647 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:30,647 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:30,649 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:30,649 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:30,650 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:30,650 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6c567b9823c64006869d468035a4942e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=36.9 K 2024-11-13T19:43:30,651 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c567b9823c64006869d468035a4942e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=814, earliestPutTs=1731527008680 2024-11-13T19:43:30,657 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e671d905bea42e488b8b5201210921d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1731527008845 2024-11-13T19:43:30,659 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288f5a35643b490e8972113f133d96aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:30,664 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/d65aa149fc5f430ea34324e8afb1a1c4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d65aa149fc5f430ea34324e8afb1a1c4 2024-11-13T19:43:30,683 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into d65aa149fc5f430ea34324e8afb1a1c4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:30,683 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:30,684 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527010566; duration=0sec 2024-11-13T19:43:30,685 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:30,685 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:30,702 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:30,703 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/d6967c8c81884fc9a05c55924504982d is 50, key is test_row_0/C:col10/1731527009121/Put/seqid=0 2024-11-13T19:43:30,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T19:43:30,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:30,743 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:30,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:30,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdc45f71939e4216b7af71b9dc844931 is 50, key is test_row_0/A:col10/1731527009602/Put/seqid=0 2024-11-13T19:43:30,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741926_1102 (size=13289) 2024-11-13T19:43:30,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527070798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527070808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527070810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527070812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741927_1103 (size=12301) 2024-11-13T19:43:30,843 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdc45f71939e4216b7af71b9dc844931 2024-11-13T19:43:30,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6495e03a4c934ba0b55f3ae6c6826d40 is 50, key is test_row_0/B:col10/1731527009602/Put/seqid=0 2024-11-13T19:43:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527070924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741928_1104 (size=12301) 2024-11-13T19:43:30,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527070933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527070934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:30,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527070944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527071134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527071145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527071154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527071154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:31,210 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/d6967c8c81884fc9a05c55924504982d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d6967c8c81884fc9a05c55924504982d 2024-11-13T19:43:31,234 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into d6967c8c81884fc9a05c55924504982d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:31,234 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:31,234 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527010567; duration=0sec 2024-11-13T19:43:31,235 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:31,235 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:31,338 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6495e03a4c934ba0b55f3ae6c6826d40 2024-11-13T19:43:31,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cf7610d8412847339c3b6c649daab248 is 50, key is test_row_0/C:col10/1731527009602/Put/seqid=0 2024-11-13T19:43:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741929_1105 (size=12301) 2024-11-13T19:43:31,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527071454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527071454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,462 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 31328a0914442b3583375df8a5770e39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527071454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527071465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527071473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:31,827 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cf7610d8412847339c3b6c649daab248 2024-11-13T19:43:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/cdc45f71939e4216b7af71b9dc844931 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931 2024-11-13T19:43:31,850 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931, entries=150, sequenceid=886, filesize=12.0 K 2024-11-13T19:43:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/6495e03a4c934ba0b55f3ae6c6826d40 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40 2024-11-13T19:43:31,870 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40, entries=150, sequenceid=886, filesize=12.0 K 2024-11-13T19:43:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cf7610d8412847339c3b6c649daab248 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248 2024-11-13T19:43:31,879 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248, entries=150, sequenceid=886, filesize=12.0 K 2024-11-13T19:43:31,882 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 31328a0914442b3583375df8a5770e39 in 1140ms, sequenceid=886, compaction requested=false 2024-11-13T19:43:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T19:43:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T19:43:31,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T19:43:31,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3050 sec 2024-11-13T19:43:31,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3240 sec 2024-11-13T19:43:31,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:31,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:32,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c92cc52a6b80424d9ab8273de652a4ff is 50, key is test_row_0/A:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:32,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527072070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527072072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527072073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527072074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741930_1106 (size=14741) 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527072207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527072207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527072210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527072210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527072444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527072444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527072453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527072454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=900 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c92cc52a6b80424d9ab8273de652a4ff 2024-11-13T19:43:32,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/3b5ca0e53e054547acaf13ee2b6f00f2 is 50, key is test_row_0/B:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:32,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741931_1107 (size=12301) 2024-11-13T19:43:32,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=900 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/3b5ca0e53e054547acaf13ee2b6f00f2 2024-11-13T19:43:32,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ccc009558a134f6b87b55845353b48c5 is 50, key is test_row_0/C:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:32,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741932_1108 (size=12301) 2024-11-13T19:43:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:43:32,724 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:32,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T19:43:32,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:32,730 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:32,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527072763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527072769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527072784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527072785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:32,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:32,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:32,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:33,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=900 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ccc009558a134f6b87b55845353b48c5 2024-11-13T19:43:33,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/c92cc52a6b80424d9ab8273de652a4ff as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff 2024-11-13T19:43:33,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff, entries=200, sequenceid=900, filesize=14.4 K 2024-11-13T19:43:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/3b5ca0e53e054547acaf13ee2b6f00f2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2 2024-11-13T19:43:33,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2, entries=150, sequenceid=900, filesize=12.0 K 2024-11-13T19:43:33,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/ccc009558a134f6b87b55845353b48c5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5 2024-11-13T19:43:33,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5, entries=150, sequenceid=900, filesize=12.0 K 2024-11-13T19:43:33,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 31328a0914442b3583375df8a5770e39 in 1265ms, sequenceid=900, compaction requested=true 2024-11-13T19:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:33,252 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:33,252 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:33,274 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:33,274 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:33,274 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,274 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/93e9785644804a90a95cc6e247e80ffa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=39.4 K 2024-11-13T19:43:33,278 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:33,278 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:33,278 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,278 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d65aa149fc5f430ea34324e8afb1a1c4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=37.0 K 2024-11-13T19:43:33,278 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e9785644804a90a95cc6e247e80ffa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:33,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527012067, max timestamp=1731527013288Num uniques 859; 2024-11-13T19:43:33,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:33,293 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d65aa149fc5f430ea34324e8afb1a1c4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:33,294 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc45f71939e4216b7af71b9dc844931, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731527009596 2024-11-13T19:43:33,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527012067, max timestamp=1731527013288Num uniques 859; 2024-11-13T19:43:33,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:33,298 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6495e03a4c934ba0b55f3ae6c6826d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731527009596 2024-11-13T19:43:33,298 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c92cc52a6b80424d9ab8273de652a4ff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:33,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527012067, max timestamp=1731527013288Num uniques 859; 2024-11-13T19:43:33,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:33,303 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5ca0e53e054547acaf13ee2b6f00f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:33,322 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:33,324 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/eb6e25d3fe3446d28027783c95d0bf2c is 50, key is test_row_0/A:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:33,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-13T19:43:33,328 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:33,329 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/b9c7dacdb8a04090b712650e47e8ed12 is 50, key is test_row_0/B:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:33,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/584ba525907e49fd9b258ca417b8dfe3 is 50, key is test_row_0/A:col10/1731527012069/Put/seqid=0 2024-11-13T19:43:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741933_1109 (size=13391) 2024-11-13T19:43:33,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,390 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/eb6e25d3fe3446d28027783c95d0bf2c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eb6e25d3fe3446d28027783c95d0bf2c 2024-11-13T19:43:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527073386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741934_1110 (size=13391) 2024-11-13T19:43:33,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527073390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527073407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527073409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741935_1111 (size=17181) 2024-11-13T19:43:33,443 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into eb6e25d3fe3446d28027783c95d0bf2c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:33,443 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:33,443 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=13, startTime=1731527013252; duration=0sec 2024-11-13T19:43:33,443 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:33,443 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:33,443 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:33,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:33,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:33,446 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,446 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d6967c8c81884fc9a05c55924504982d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=37.0 K 2024-11-13T19:43:33,449 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6967c8c81884fc9a05c55924504982d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=864, earliestPutTs=1731527009121 2024-11-13T19:43:33,453 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7610d8412847339c3b6c649daab248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=886, earliestPutTs=1731527009596 2024-11-13T19:43:33,454 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc009558a134f6b87b55845353b48c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:33,476 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:33,477 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/d2d14736dcd64788be473860990ba8a5 is 50, key is test_row_0/C:col10/1731527010809/Put/seqid=0 2024-11-13T19:43:33,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49228 deadline: 1731527073483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,491 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 31328a0914442b3583375df8a5770e39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741936_1112 (size=13391) 2024-11-13T19:43:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527073518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527073524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527073524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527073573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527073776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527073783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527073783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527073785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:33,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:33,833 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/b9c7dacdb8a04090b712650e47e8ed12 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/b9c7dacdb8a04090b712650e47e8ed12 2024-11-13T19:43:33,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/584ba525907e49fd9b258ca417b8dfe3 2024-11-13T19:43:33,864 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into b9c7dacdb8a04090b712650e47e8ed12(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:33,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:33,865 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=13, startTime=1731527013252; duration=0sec 2024-11-13T19:43:33,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:33,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:33,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/c304803723ca4810be246d7736fa5949 is 50, key is test_row_0/B:col10/1731527012069/Put/seqid=0 2024-11-13T19:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:33,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:33,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:33,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741937_1113 (size=12301) 2024-11-13T19:43:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:33,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/d2d14736dcd64788be473860990ba8a5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d2d14736dcd64788be473860990ba8a5 2024-11-13T19:43:33,979 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into d2d14736dcd64788be473860990ba8a5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:33,979 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:33,979 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=13, startTime=1731527013254; duration=0sec 2024-11-13T19:43:33,979 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:33,979 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:34,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527074097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527074104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527074104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527074113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/c304803723ca4810be246d7736fa5949 2024-11-13T19:43:34,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1e14e114a2a744e5a6b757cf1044701e is 50, key is test_row_0/C:col10/1731527012069/Put/seqid=0 2024-11-13T19:43:34,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741938_1114 (size=12301) 2024-11-13T19:43:34,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527074618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527074618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527074618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527074638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:34,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1e14e114a2a744e5a6b757cf1044701e 2024-11-13T19:43:34,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/584ba525907e49fd9b258ca417b8dfe3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3 2024-11-13T19:43:34,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3, entries=250, sequenceid=928, filesize=16.8 K 2024-11-13T19:43:34,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/c304803723ca4810be246d7736fa5949 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949 2024-11-13T19:43:34,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. as already flushing 2024-11-13T19:43:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:34,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:34,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949, entries=150, sequenceid=928, filesize=12.0 K 2024-11-13T19:43:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/1e14e114a2a744e5a6b757cf1044701e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e 2024-11-13T19:43:34,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e, entries=150, sequenceid=928, filesize=12.0 K 2024-11-13T19:43:34,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for 31328a0914442b3583375df8a5770e39 in 1554ms, sequenceid=928, compaction requested=false 2024-11-13T19:43:34,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:35,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:43:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:35,004 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:43:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/4413c98ffa1744ae94450862682d22a0 is 50, key is test_row_0/A:col10/1731527013407/Put/seqid=0 2024-11-13T19:43:35,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741939_1115 (size=12301) 2024-11-13T19:43:35,057 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=950 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/4413c98ffa1744ae94450862682d22a0 2024-11-13T19:43:35,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aaeac4d0fea44c9a9157ce814853082b is 50, key is test_row_0/B:col10/1731527013407/Put/seqid=0 2024-11-13T19:43:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741940_1116 (size=12301) 2024-11-13T19:43:35,095 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=950 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aaeac4d0fea44c9a9157ce814853082b 2024-11-13T19:43:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/645c5d85cc514758a6c684fa8118f412 is 50, key is test_row_0/C:col10/1731527013407/Put/seqid=0 2024-11-13T19:43:35,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741941_1117 (size=12301) 2024-11-13T19:43:35,582 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=950 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/645c5d85cc514758a6c684fa8118f412 2024-11-13T19:43:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/4413c98ffa1744ae94450862682d22a0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0 2024-11-13T19:43:35,611 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0, entries=150, sequenceid=950, filesize=12.0 K 2024-11-13T19:43:35,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/aaeac4d0fea44c9a9157ce814853082b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b 2024-11-13T19:43:35,620 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b, entries=150, sequenceid=950, filesize=12.0 K 2024-11-13T19:43:35,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/645c5d85cc514758a6c684fa8118f412 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412 2024-11-13T19:43:35,635 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412, entries=150, sequenceid=950, filesize=12.0 K 2024-11-13T19:43:35,636 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 31328a0914442b3583375df8a5770e39 in 632ms, sequenceid=950, compaction requested=true 2024-11-13T19:43:35,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:35,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T19:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T19:43:35,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T19:43:35,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9090 sec 2024-11-13T19:43:35,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.9210 sec 2024-11-13T19:43:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:35,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:35,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/dde6a8e9d390487db624020c409cbd23 is 50, key is test_row_0/A:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:35,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741942_1118 (size=14741) 2024-11-13T19:43:35,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/dde6a8e9d390487db624020c409cbd23 2024-11-13T19:43:35,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527015697, max timestamp=1731527015722Num uniques 859; 2024-11-13T19:43:35,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527015697, max timestamp=1731527015722Num uniques 859; 2024-11-13T19:43:35,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:35,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527015697, max timestamp=1731527015722Num uniques 859; 2024-11-13T19:43:35,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:35,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/81b7fac347c146b2990a93e017323aba is 50, key is test_row_0/B:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741943_1119 (size=12301) 2024-11-13T19:43:35,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/81b7fac347c146b2990a93e017323aba 2024-11-13T19:43:35,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/fb9a2dab85564c1db76159039c4a00b4 is 50, key is test_row_0/C:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:35,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527075764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527075765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527075772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527075774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741944_1120 (size=12301) 2024-11-13T19:43:35,868 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T19:43:35,868 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:35,870 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T19:43:35,871 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:35,873 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T19:43:35,873 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:35,876 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,876 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:35,877 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T19:43:35,877 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:35,877 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,877 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:35,877 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,877 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:35,878 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,878 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:35,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:35,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527075884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527075884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527075884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:35,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527075893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49206 deadline: 1731527076094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49210 deadline: 1731527076094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49222 deadline: 1731527076094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1731527076105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:43:36,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/fb9a2dab85564c1db76159039c4a00b4 2024-11-13T19:43:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/dde6a8e9d390487db624020c409cbd23 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23 2024-11-13T19:43:36,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23, entries=200, sequenceid=960, filesize=14.4 K 2024-11-13T19:43:36,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/81b7fac347c146b2990a93e017323aba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba 2024-11-13T19:43:36,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba, entries=150, sequenceid=960, filesize=12.0 K 2024-11-13T19:43:36,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/fb9a2dab85564c1db76159039c4a00b4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4 2024-11-13T19:43:36,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4, entries=150, sequenceid=960, filesize=12.0 K 2024-11-13T19:43:36,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 31328a0914442b3583375df8a5770e39 in 615ms, sequenceid=960, compaction requested=true 2024-11-13T19:43:36,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:36,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:36,278 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31328a0914442b3583375df8a5770e39:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:36,279 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:36,279 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/A is initiating minor compaction (all files) 2024-11-13T19:43:36,279 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/A in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:36,279 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eb6e25d3fe3446d28027783c95d0bf2c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=56.3 K 2024-11-13T19:43:36,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6e25d3fe3446d28027783c95d0bf2c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:36,281 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:36,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584ba525907e49fd9b258ca417b8dfe3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731527012069 2024-11-13T19:43:36,282 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4413c98ffa1744ae94450862682d22a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=950, earliestPutTs=1731527013375 2024-11-13T19:43:36,282 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde6a8e9d390487db624020c409cbd23, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1731527015647 2024-11-13T19:43:36,285 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:36,285 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/B is initiating minor compaction (all files) 2024-11-13T19:43:36,285 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/B in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:36,286 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/b9c7dacdb8a04090b712650e47e8ed12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=49.1 K 2024-11-13T19:43:36,287 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b9c7dacdb8a04090b712650e47e8ed12, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:36,288 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c304803723ca4810be246d7736fa5949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731527012069 2024-11-13T19:43:36,289 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting aaeac4d0fea44c9a9157ce814853082b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=950, earliestPutTs=1731527013375 2024-11-13T19:43:36,289 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b7fac347c146b2990a93e017323aba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1731527015652 2024-11-13T19:43:36,318 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:36,319 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/994f2a1b32a74c378866b0728842e498 is 50, key is test_row_0/B:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:36,323 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:36,324 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/ad2a536428ca42f0a5fa8e19759f950f is 50, key is test_row_0/A:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741945_1121 (size=13527) 2024-11-13T19:43:36,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741946_1122 (size=13527) 2024-11-13T19:43:36,373 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/ad2a536428ca42f0a5fa8e19759f950f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/ad2a536428ca42f0a5fa8e19759f950f 2024-11-13T19:43:36,381 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/A of 31328a0914442b3583375df8a5770e39 into ad2a536428ca42f0a5fa8e19759f950f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:36,381 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:36,381 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/A, priority=12, startTime=1731527016277; duration=0sec 2024-11-13T19:43:36,381 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:36,381 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:A 2024-11-13T19:43:36,381 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:36,382 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:36,382 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 31328a0914442b3583375df8a5770e39/C is initiating minor compaction (all files) 2024-11-13T19:43:36,383 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31328a0914442b3583375df8a5770e39/C in TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:36,383 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d2d14736dcd64788be473860990ba8a5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp, totalSize=49.1 K 2024-11-13T19:43:36,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d14736dcd64788be473860990ba8a5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=900, earliestPutTs=1731527010803 2024-11-13T19:43:36,384 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e14e114a2a744e5a6b757cf1044701e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731527012069 2024-11-13T19:43:36,384 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 645c5d85cc514758a6c684fa8118f412, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=950, earliestPutTs=1731527013375 2024-11-13T19:43:36,385 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9a2dab85564c1db76159039c4a00b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1731527015652 2024-11-13T19:43:36,397 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31328a0914442b3583375df8a5770e39#C#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:36,398 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/5a78ce09e6de45b58a005dbf67e0e399 is 50, key is test_row_0/C:col10/1731527015652/Put/seqid=0 2024-11-13T19:43:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527015761, max timestamp=1731527016405Num uniques 1068; 2024-11-13T19:43:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527015761, max timestamp=1731527016405Num uniques 1068; 2024-11-13T19:43:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T19:43:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T19:43:36,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527015761, max timestamp=1731527016405Num uniques 1068; 2024-11-13T19:43:36,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T19:43:36,413 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T19:43:36,413 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:36,413 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,413 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,413 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T19:43:36,413 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:36,413 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:36,413 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:36,417 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T19:43:36,417 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:36,417 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,417 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:36,419 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T19:43:36,419 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:36,419 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,419 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:36,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:36,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741947_1123 (size=13527) 2024-11-13T19:43:36,475 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/5a78ce09e6de45b58a005dbf67e0e399 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/5a78ce09e6de45b58a005dbf67e0e399 2024-11-13T19:43:36,487 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/C of 31328a0914442b3583375df8a5770e39 into 5a78ce09e6de45b58a005dbf67e0e399(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:36,487 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:36,487 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/C, priority=12, startTime=1731527016278; duration=0sec 2024-11-13T19:43:36,491 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:36,491 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:C 2024-11-13T19:43:36,796 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/994f2a1b32a74c378866b0728842e498 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/994f2a1b32a74c378866b0728842e498 2024-11-13T19:43:36,815 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31328a0914442b3583375df8a5770e39/B of 31328a0914442b3583375df8a5770e39 into 994f2a1b32a74c378866b0728842e498(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:36,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:36,815 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39., storeName=31328a0914442b3583375df8a5770e39/B, priority=12, startTime=1731527016278; duration=0sec 2024-11-13T19:43:36,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:36,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31328a0914442b3583375df8a5770e39:B 2024-11-13T19:43:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:43:36,907 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:37,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31328a0914442b3583375df8a5770e39 3/3 column families, dataSize=53.67 KB heapSize=140.39 KB 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=A 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=B 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 31328a0914442b3583375df8a5770e39, store=C 2024-11-13T19:43:37,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:37,542 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T19:43:37,542 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:43:37,542 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:37,542 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:37,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:37,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 192 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 195 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1212 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1141 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 598 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1779 rows 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 598 2024-11-13T19:43:37,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1794 rows 2024-11-13T19:43:37,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:43:37,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:43:37,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:37,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:37,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:43:37,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:43:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:37,562 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527017562"}]},"ts":"1731527017562"} 2024-11-13T19:43:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:43:37,565 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:43:37,565 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:43:37,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:43:37,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, UNASSIGN}] 2024-11-13T19:43:37,575 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, UNASSIGN 2024-11-13T19:43:37,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/50d7e439e2804255b67622d15a7392ea is 50, key is test_row_0/A:col10/1731527016415/Put/seqid=0 2024-11-13T19:43:37,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=31328a0914442b3583375df8a5770e39, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:37,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, UNASSIGN because future has completed 2024-11-13T19:43:37,582 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:43:37,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741948_1124 (size=12301) 2024-11-13T19:43:37,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1004 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/50d7e439e2804255b67622d15a7392ea 2024-11-13T19:43:37,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/02de76ec51a84b07a50600a0fe1ec8dc is 50, key is test_row_0/B:col10/1731527016415/Put/seqid=0 2024-11-13T19:43:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:43:37,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741949_1125 (size=12301) 2024-11-13T19:43:37,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1004 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/02de76ec51a84b07a50600a0fe1ec8dc 2024-11-13T19:43:37,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cdfd4768fe044767b6d92458b4df9d44 is 50, key is test_row_0/C:col10/1731527016415/Put/seqid=0 2024-11-13T19:43:37,706 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:43:37,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741950_1126 (size=12301) 2024-11-13T19:43:37,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1004 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cdfd4768fe044767b6d92458b4df9d44 2024-11-13T19:43:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/A/50d7e439e2804255b67622d15a7392ea as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/50d7e439e2804255b67622d15a7392ea 2024-11-13T19:43:37,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/50d7e439e2804255b67622d15a7392ea, entries=150, sequenceid=1004, filesize=12.0 K 2024-11-13T19:43:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/B/02de76ec51a84b07a50600a0fe1ec8dc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/02de76ec51a84b07a50600a0fe1ec8dc 2024-11-13T19:43:37,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/02de76ec51a84b07a50600a0fe1ec8dc, entries=150, sequenceid=1004, filesize=12.0 K 2024-11-13T19:43:37,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/.tmp/C/cdfd4768fe044767b6d92458b4df9d44 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cdfd4768fe044767b6d92458b4df9d44 2024-11-13T19:43:37,777 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:37,778 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:43:37,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cdfd4768fe044767b6d92458b4df9d44, entries=150, sequenceid=1004, filesize=12.0 K 2024-11-13T19:43:37,779 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 31328a0914442b3583375df8a5770e39, disabling compactions & flushes 2024-11-13T19:43:37,779 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:37,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~140.34 KB/143712, currentSize=0 B/0 for 31328a0914442b3583375df8a5770e39 in 239ms, sequenceid=1004, compaction requested=false 2024-11-13T19:43:37,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31328a0914442b3583375df8a5770e39: 2024-11-13T19:43:37,780 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:37,780 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:37,780 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. after waiting 0 ms 2024-11-13T19:43:37,780 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:37,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/9ff15312f661405da03f18398b0226a4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/210d471d9b29485593fa4194c94a8c12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/772bbd8deb68499b9caafcc0a300404e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d24905f77234b43a55bd1796c98b74f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4f4e95148f0043a4a12d29e29ad4c4b3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/d1e2a5e6c0544f8f9ef61d338111f741, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/89cc145aed214167bbdb1fcf4116f5fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eec02e5e1eb54e689e1cd886b899cfbf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/93e9785644804a90a95cc6e247e80ffa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eb6e25d3fe3446d28027783c95d0bf2c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23] to archive 2024-11-13T19:43:37,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:43:37,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/00b84d70ae004d4b877a49a4f9a77946 2024-11-13T19:43:37,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10c508062930424a9679774c964ddf9c 2024-11-13T19:43:37,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c668c37816a840a1ad6c0bdbddf0c5a6 2024-11-13T19:43:37,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/674218d886e44343bd9206201d4ec695 2024-11-13T19:43:37,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/9ff15312f661405da03f18398b0226a4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/9ff15312f661405da03f18398b0226a4 2024-11-13T19:43:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:43:37,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/98026d98593b421d804e45d71abaac60 2024-11-13T19:43:37,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/210d471d9b29485593fa4194c94a8c12 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/210d471d9b29485593fa4194c94a8c12 2024-11-13T19:43:37,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1ba3656721514434a24cbf595116206b 2024-11-13T19:43:37,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/543cda9f746742cfb9518f9438317263 2024-11-13T19:43:37,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/343339dda8b44862ba9a5917376cbee6 2024-11-13T19:43:37,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/772bbd8deb68499b9caafcc0a300404e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/772bbd8deb68499b9caafcc0a300404e 2024-11-13T19:43:37,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdf46a5d5de14ffc84acd559e6641e16 2024-11-13T19:43:37,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/6a36a87f343940b2ac6eacbd5cae527e 2024-11-13T19:43:37,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d24905f77234b43a55bd1796c98b74f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d24905f77234b43a55bd1796c98b74f 2024-11-13T19:43:37,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/3f697f2cd41249ab8888efcfc6b2e744 2024-11-13T19:43:37,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/961d63fac60b4334a7a15dd6270f09bf 2024-11-13T19:43:37,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4f4e95148f0043a4a12d29e29ad4c4b3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4f4e95148f0043a4a12d29e29ad4c4b3 2024-11-13T19:43:37,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/7b75e0f5509a41b8900e53191c09f46d 2024-11-13T19:43:37,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f2e0a2e6d586484dbde5bbf147006995 2024-11-13T19:43:37,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/edd2cbba0e78482998ba332b335e2857 2024-11-13T19:43:37,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/d1e2a5e6c0544f8f9ef61d338111f741 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/d1e2a5e6c0544f8f9ef61d338111f741 2024-11-13T19:43:37,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/1d591f3a78184c79b4ec20ec42dc8744 2024-11-13T19:43:37,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/89cc145aed214167bbdb1fcf4116f5fd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/89cc145aed214167bbdb1fcf4116f5fd 2024-11-13T19:43:37,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/229af55a792c499ca1f5d48e4c3b05a6 2024-11-13T19:43:37,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/10a82f540a584fd0ab4fe5792209abc3 2024-11-13T19:43:37,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/828b70c3ee214fd68907c1dd00d60289 2024-11-13T19:43:37,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eec02e5e1eb54e689e1cd886b899cfbf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eec02e5e1eb54e689e1cd886b899cfbf 2024-11-13T19:43:37,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/f920c3f9ebf3461fb7c930f3bc3e3468 2024-11-13T19:43:37,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/2af47da124af4505a8eeb52fc4f3dfd2 2024-11-13T19:43:37,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/93e9785644804a90a95cc6e247e80ffa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/93e9785644804a90a95cc6e247e80ffa 2024-11-13T19:43:37,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/cdc45f71939e4216b7af71b9dc844931 2024-11-13T19:43:37,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/c92cc52a6b80424d9ab8273de652a4ff 2024-11-13T19:43:37,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eb6e25d3fe3446d28027783c95d0bf2c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/eb6e25d3fe3446d28027783c95d0bf2c 2024-11-13T19:43:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/584ba525907e49fd9b258ca417b8dfe3 2024-11-13T19:43:37,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/4413c98ffa1744ae94450862682d22a0 2024-11-13T19:43:38,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/dde6a8e9d390487db624020c409cbd23 2024-11-13T19:43:38,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/f0afbed517244edab56c157d305134a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6c80cb22746a4f2bab9f0f40dc67ac83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/46eddb8ec2eb4930acb8ea6216c82371, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/220ca12e1cf84c529ae47934dcc0cc2e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d04a28f7fd91432ebe89e4b13940cd51, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ef04e947a3a9454a9f9eaa5be7cf2a8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/68a2cc830d3646398de3101fd810ba1d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3f8da903304a4ffab295f231adb47ac1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d65aa149fc5f430ea34324e8afb1a1c4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/b9c7dacdb8a04090b712650e47e8ed12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba] to archive 2024-11-13T19:43:38,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:43:38,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cedb0ddc68d74a688f10d0892304ca4f 2024-11-13T19:43:38,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f546f3533be4f5eb7a9c6e0e7a78f84 2024-11-13T19:43:38,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6235d6dec3ac40b7a4b20f8fb88a87d7 2024-11-13T19:43:38,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/f0afbed517244edab56c157d305134a3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/f0afbed517244edab56c157d305134a3 2024-11-13T19:43:38,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/69d89a006bc9489b9dae225ba1a6156f 2024-11-13T19:43:38,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e267b7f8cb6e48ada4f7a26343f9e72f 2024-11-13T19:43:38,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6c80cb22746a4f2bab9f0f40dc67ac83 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6c80cb22746a4f2bab9f0f40dc67ac83 2024-11-13T19:43:38,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/0f4aa884554f4c01b5a73a3c8107d006 2024-11-13T19:43:38,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/a54062f9a6494755971251dcebd6066e 2024-11-13T19:43:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:43:38,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/9071df797b7d49a29b70f407c14aad55 2024-11-13T19:43:38,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/46eddb8ec2eb4930acb8ea6216c82371 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/46eddb8ec2eb4930acb8ea6216c82371 2024-11-13T19:43:38,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/fd126972e6fe4419a3dd36a5d94d4126 2024-11-13T19:43:38,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/8a735c83da68426ebc6e195269759892 2024-11-13T19:43:38,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/220ca12e1cf84c529ae47934dcc0cc2e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/220ca12e1cf84c529ae47934dcc0cc2e 2024-11-13T19:43:38,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aec518a8763943fca1e6830aff5b3802 2024-11-13T19:43:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/7da5e6617add4db68c6d86435e5b683c 2024-11-13T19:43:38,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d04a28f7fd91432ebe89e4b13940cd51 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d04a28f7fd91432ebe89e4b13940cd51 2024-11-13T19:43:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/167b7615d02f474fbe661a4e566ebee8 2024-11-13T19:43:38,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ee8e0d28e12b42aca944cfced1a026e6 2024-11-13T19:43:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ef04e947a3a9454a9f9eaa5be7cf2a8a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/ef04e947a3a9454a9f9eaa5be7cf2a8a 2024-11-13T19:43:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/e6f9060f439744259d42584bc06c9f48 2024-11-13T19:43:38,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/cc0c8b7a38de4fabb0fbca7466f706bd 2024-11-13T19:43:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/68a2cc830d3646398de3101fd810ba1d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/68a2cc830d3646398de3101fd810ba1d 2024-11-13T19:43:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/1880e3b7e31d45e6802f52cefff0f7f7 2024-11-13T19:43:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/91fcc59ce14f47ceaf627cfc31a7651a 2024-11-13T19:43:38,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3f8da903304a4ffab295f231adb47ac1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3f8da903304a4ffab295f231adb47ac1 2024-11-13T19:43:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/96d6b46b7eda4f2d83c3992eb7fc594b 2024-11-13T19:43:38,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/5073e973a7554fb7b3ca55e14d98decb 2024-11-13T19:43:38,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d65aa149fc5f430ea34324e8afb1a1c4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/d65aa149fc5f430ea34324e8afb1a1c4 2024-11-13T19:43:38,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/927e0845e62a4ba3aa9f8eb46731f9f3 2024-11-13T19:43:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/6495e03a4c934ba0b55f3ae6c6826d40 2024-11-13T19:43:38,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/b9c7dacdb8a04090b712650e47e8ed12 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/b9c7dacdb8a04090b712650e47e8ed12 2024-11-13T19:43:38,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/3b5ca0e53e054547acaf13ee2b6f00f2 2024-11-13T19:43:38,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/c304803723ca4810be246d7736fa5949 2024-11-13T19:43:38,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/aaeac4d0fea44c9a9157ce814853082b 2024-11-13T19:43:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/81b7fac347c146b2990a93e017323aba 2024-11-13T19:43:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/77cabdb8330d4914b366b7f97f2033db, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/97c8199c57734e5ab97687ba2ee0ce1f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/271a5f26b4cb4702bce97f00a5306049, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1c2577f4d3414c75a226ca2c5f3625ac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ddd364a19bd14fe2b557565f74989c7f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0b38e0f8a5b34b4dad143d723c08736b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/f8175fabe43742d589de7ba36cf54064, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6c567b9823c64006869d468035a4942e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d6967c8c81884fc9a05c55924504982d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d2d14736dcd64788be473860990ba8a5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4] to archive 2024-11-13T19:43:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:43:38,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ffe3aeb5c63e469d9ad5d778c7b31d03 2024-11-13T19:43:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/aeea3c3f37f44c0cab8478fef1fa6199 2024-11-13T19:43:38,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/e01d3715deea4416a99930d207a34e83 2024-11-13T19:43:38,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/88cb8973cc534cc69ffe7b529d8205a3 2024-11-13T19:43:38,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/77cabdb8330d4914b366b7f97f2033db to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/77cabdb8330d4914b366b7f97f2033db 2024-11-13T19:43:38,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8fdeee9a71204fd1b18ff8767f094ec7 2024-11-13T19:43:38,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/69305634d70d48b29aa87842b37993df 2024-11-13T19:43:38,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/97c8199c57734e5ab97687ba2ee0ce1f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/97c8199c57734e5ab97687ba2ee0ce1f 2024-11-13T19:43:38,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3780312960934d27897c643908459186 2024-11-13T19:43:38,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/c627751ba5bd494d9e77ef72a5c35dd3 2024-11-13T19:43:38,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/271a5f26b4cb4702bce97f00a5306049 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/271a5f26b4cb4702bce97f00a5306049 2024-11-13T19:43:38,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/bcad99a5773d4db9b0101c23ad6c01ff 2024-11-13T19:43:38,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cc7afefd60ca4e53adbbd30d007e6edd 2024-11-13T19:43:38,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1c2577f4d3414c75a226ca2c5f3625ac to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1c2577f4d3414c75a226ca2c5f3625ac 2024-11-13T19:43:38,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1fe7e9df895f4fbb9d3fd48ae62edb1f 2024-11-13T19:43:38,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/8dd3257c30d4451095682f036b77c862 2024-11-13T19:43:38,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ddd364a19bd14fe2b557565f74989c7f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ddd364a19bd14fe2b557565f74989c7f 2024-11-13T19:43:38,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b493f516e9014e488ef23100641d58c2 2024-11-13T19:43:38,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/3bff4bebf7b8459a8983fa804d0a1bb5 2024-11-13T19:43:38,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0b38e0f8a5b34b4dad143d723c08736b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0b38e0f8a5b34b4dad143d723c08736b 2024-11-13T19:43:38,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/0410f690b67c4a44877fabbd8d52a8ba 2024-11-13T19:43:38,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/16f86515b5394c96bc75330d636795eb 2024-11-13T19:43:38,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/f8175fabe43742d589de7ba36cf54064 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/f8175fabe43742d589de7ba36cf54064 2024-11-13T19:43:38,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6964908d1c944060b6185d33fd9456c5 2024-11-13T19:43:38,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/a0819b41340a42a084e11860ac073b48 2024-11-13T19:43:38,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6c567b9823c64006869d468035a4942e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6c567b9823c64006869d468035a4942e 2024-11-13T19:43:38,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/b7d94d86bf214399b5202ce9e349b7a6 2024-11-13T19:43:38,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/6e671d905bea42e488b8b5201210921d 2024-11-13T19:43:38,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d6967c8c81884fc9a05c55924504982d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d6967c8c81884fc9a05c55924504982d 2024-11-13T19:43:38,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/288f5a35643b490e8972113f133d96aa 2024-11-13T19:43:38,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cf7610d8412847339c3b6c649daab248 2024-11-13T19:43:38,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d2d14736dcd64788be473860990ba8a5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/d2d14736dcd64788be473860990ba8a5 2024-11-13T19:43:38,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/ccc009558a134f6b87b55845353b48c5 2024-11-13T19:43:38,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/1e14e114a2a744e5a6b757cf1044701e 2024-11-13T19:43:38,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/645c5d85cc514758a6c684fa8118f412 2024-11-13T19:43:38,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/fb9a2dab85564c1db76159039c4a00b4 2024-11-13T19:43:38,337 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/recovered.edits/1007.seqid, newMaxSeqId=1007, maxSeqId=1 2024-11-13T19:43:38,339 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39. 2024-11-13T19:43:38,339 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 31328a0914442b3583375df8a5770e39: Waiting for close lock at 1731527017779Running coprocessor pre-close hooks at 1731527017779Disabling compacts and flushes for region at 1731527017779Disabling writes for close at 1731527017780 (+1 ms)Writing region close event to WAL at 1731527018331 (+551 ms)Running coprocessor post-close hooks at 1731527018338 (+7 ms)Closed at 1731527018339 (+1 ms) 2024-11-13T19:43:38,342 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 31328a0914442b3583375df8a5770e39 2024-11-13T19:43:38,343 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=31328a0914442b3583375df8a5770e39, regionState=CLOSED 2024-11-13T19:43:38,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:43:38,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T19:43:38,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 31328a0914442b3583375df8a5770e39, server=54d0784d1943,36227,1731526989835 in 767 msec 2024-11-13T19:43:38,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T19:43:38,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31328a0914442b3583375df8a5770e39, UNASSIGN in 779 msec 2024-11-13T19:43:38,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T19:43:38,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 790 msec 2024-11-13T19:43:38,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527018363"}]},"ts":"1731527018363"} 2024-11-13T19:43:38,365 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:43:38,365 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:43:38,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 817 msec 2024-11-13T19:43:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:43:38,704 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:38,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:43:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,713 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:43:38,714 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,719 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39 2024-11-13T19:43:38,723 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/recovered.edits] 2024-11-13T19:43:38,727 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/50d7e439e2804255b67622d15a7392ea to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/50d7e439e2804255b67622d15a7392ea 2024-11-13T19:43:38,729 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/ad2a536428ca42f0a5fa8e19759f950f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/A/ad2a536428ca42f0a5fa8e19759f950f 2024-11-13T19:43:38,732 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/02de76ec51a84b07a50600a0fe1ec8dc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/02de76ec51a84b07a50600a0fe1ec8dc 2024-11-13T19:43:38,734 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/994f2a1b32a74c378866b0728842e498 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/B/994f2a1b32a74c378866b0728842e498 2024-11-13T19:43:38,736 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/5a78ce09e6de45b58a005dbf67e0e399 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/5a78ce09e6de45b58a005dbf67e0e399 2024-11-13T19:43:38,737 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cdfd4768fe044767b6d92458b4df9d44 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/C/cdfd4768fe044767b6d92458b4df9d44 2024-11-13T19:43:38,740 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/recovered.edits/1007.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39/recovered.edits/1007.seqid 2024-11-13T19:43:38,741 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/31328a0914442b3583375df8a5770e39 2024-11-13T19:43:38,741 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:43:38,746 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T19:43:38,757 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:43:38,795 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:43:38,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:43:38,797 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527018797"}]},"ts":"9223372036854775807"} 2024-11-13T19:43:38,802 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:43:38,802 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 31328a0914442b3583375df8a5770e39, NAME => 'TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:43:38,803 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:43:38,803 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527018803"}]},"ts":"9223372036854775807"} 2024-11-13T19:43:38,805 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:43:38,807 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-13T19:43:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:43:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:43:38,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:38,846 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/54d0784d1943:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;54d0784d1943:36227-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_543150073_22 at /127.0.0.1:34420 [Waiting for operation #239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1517952646_22 at /127.0.0.1:34502 [Waiting for operation #217] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_543150073_22 at /127.0.0.1:43316 [Waiting for operation #260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=897 (was 610) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=328 (was 1448) 2024-11-13T19:43:38,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=897, ProcessCount=11, AvailableMemoryMB=325 2024-11-13T19:43:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:43:38,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:43:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:38,862 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:43:38,863 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:38,863 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-13T19:43:38,864 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:43:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T19:43:38,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741951_1127 (size=1061) 2024-11-13T19:43:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T19:43:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T19:43:39,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:43:39,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741952_1128 (size=53) 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing da41fb4884d60614ee8b9d1f2d3eb0fa, disabling compactions & flushes 2024-11-13T19:43:39,312 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. after waiting 0 ms 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,312 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,312 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for da41fb4884d60614ee8b9d1f2d3eb0fa: Waiting for close lock at 1731527019312Disabling compacts and flushes for region at 1731527019312Disabling writes for close at 1731527019312Writing region close event to WAL at 1731527019312Closed at 1731527019312 2024-11-13T19:43:39,314 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:43:39,315 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731527019314"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527019314"}]},"ts":"1731527019314"} 2024-11-13T19:43:39,319 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:43:39,320 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:43:39,321 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527019321"}]},"ts":"1731527019321"} 2024-11-13T19:43:39,323 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:43:39,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, ASSIGN}] 2024-11-13T19:43:39,325 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, ASSIGN 2024-11-13T19:43:39,326 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:43:39,328 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:43:39,477 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:39,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, ASSIGN because future has completed 2024-11-13T19:43:39,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T19:43:39,636 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,636 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:43:39,636 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,637 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:39,637 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,637 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,638 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,640 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:39,641 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName A 2024-11-13T19:43:39,641 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:39,641 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:39,642 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,643 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:39,643 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName B 2024-11-13T19:43:39,643 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:39,643 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:39,643 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,645 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:39,645 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName C 2024-11-13T19:43:39,645 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:39,645 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:39,645 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,645 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,646 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,646 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,648 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,648 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,648 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:43:39,649 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,651 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:43:39,652 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened da41fb4884d60614ee8b9d1f2d3eb0fa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75415015, jitterRate=0.1237712949514389}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:43:39,652 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:39,653 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for da41fb4884d60614ee8b9d1f2d3eb0fa: Running coprocessor pre-open hook at 1731527019637Writing region info on filesystem at 1731527019637Initializing all the Stores at 1731527019638 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527019638Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527019638Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527019638Cleaning up temporary data from old regions at 1731527019648 (+10 ms)Running coprocessor post-open hooks at 1731527019652 (+4 ms)Region opened successfully at 1731527019653 (+1 ms) 2024-11-13T19:43:39,654 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., pid=26, masterSystemTime=1731527019632 2024-11-13T19:43:39,656 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,656 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:39,657 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:39,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:43:39,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T19:43:39,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 in 180 msec 2024-11-13T19:43:39,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T19:43:39,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, ASSIGN in 340 msec 2024-11-13T19:43:39,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:43:39,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527019665"}]},"ts":"1731527019665"} 2024-11-13T19:43:39,667 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:43:39,668 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:43:39,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 808 msec 2024-11-13T19:43:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T19:43:40,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:40,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67353d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,010 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c68170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,011 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:40,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:40,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:43:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:43:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3accb36a 2024-11-13T19:43:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:43:40,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:43:40,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:43:40,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:43:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:43:40,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741953_1129 (size=1097) 2024-11-13T19:43:40,443 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T19:43:40,443 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T19:43:40,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:43:40,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE}] 2024-11-13T19:43:40,455 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE 2024-11-13T19:43:40,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:40,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE because future has completed 2024-11-13T19:43:40,458 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:43:40,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:40,611 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:43:40,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing da41fb4884d60614ee8b9d1f2d3eb0fa, disabling compactions & flushes 2024-11-13T19:43:40,611 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. after waiting 0 ms 2024-11-13T19:43:40,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,615 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:43:40,615 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,616 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for da41fb4884d60614ee8b9d1f2d3eb0fa: Waiting for close lock at 1731527020611Running coprocessor pre-close hooks at 1731527020611Disabling compacts and flushes for region at 1731527020611Disabling writes for close at 1731527020611Writing region close event to WAL at 1731527020611Running coprocessor post-close hooks at 1731527020615 (+4 ms)Closed at 1731527020615 2024-11-13T19:43:40,616 WARN [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: da41fb4884d60614ee8b9d1f2d3eb0fa to self. 2024-11-13T19:43:40,618 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=CLOSED 2024-11-13T19:43:40,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:43:40,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T19:43:40,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 in 163 msec 2024-11-13T19:43:40,624 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE; state=CLOSED, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=true 2024-11-13T19:43:40,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:40,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE because future has completed 2024-11-13T19:43:40,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:43:40,933 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,933 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:43:40,933 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,933 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:43:40,933 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,934 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,938 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,939 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:40,944 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName A 2024-11-13T19:43:40,945 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:40,946 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:40,946 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,947 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:40,947 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName B 2024-11-13T19:43:40,947 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:40,947 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:40,948 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,948 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:43:40,948 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da41fb4884d60614ee8b9d1f2d3eb0fa columnFamilyName C 2024-11-13T19:43:40,948 DEBUG [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:40,949 INFO [StoreOpener-da41fb4884d60614ee8b9d1f2d3eb0fa-1 {}] regionserver.HStore(327): Store=da41fb4884d60614ee8b9d1f2d3eb0fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:43:40,949 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,949 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,950 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,951 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,952 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,952 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,952 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:43:40,954 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,955 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened da41fb4884d60614ee8b9d1f2d3eb0fa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67509413, jitterRate=0.005968645215034485}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:43:40,955 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:40,956 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for da41fb4884d60614ee8b9d1f2d3eb0fa: Running coprocessor pre-open hook at 1731527020934Writing region info on filesystem at 1731527020934Initializing all the Stores at 1731527020934Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527020934Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527020935 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527020935Cleaning up temporary data from old regions at 1731527020952 (+17 ms)Running coprocessor post-open hooks at 1731527020955 (+3 ms)Region opened successfully at 1731527020956 (+1 ms) 2024-11-13T19:43:40,957 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., pid=31, masterSystemTime=1731527020929 2024-11-13T19:43:40,959 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,959 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:40,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=OPEN, openSeqNum=5, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:43:40,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:43:40,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T19:43:40,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 in 185 msec 2024-11-13T19:43:40,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T19:43:40,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, REOPEN/MOVE in 510 msec 2024-11-13T19:43:40,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T19:43:40,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-13T19:43:40,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-11-13T19:43:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:43:40,979 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:40,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1189ecb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,983 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab3e094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,985 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ee04c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,988 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5e96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,989 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba6675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,991 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b5a131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,993 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a30b61d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dac5d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:40,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649b6808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:40,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:40,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e8ab95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:41,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:41,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479f8582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:41,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:41,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b434e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:41,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:41,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:41,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341602f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:41,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:41,005 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:41,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:41,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:41,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57bfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:43:41,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:43:41,008 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:43:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:43:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:43:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cedb6d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:43:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:43:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:43:41,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:43:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e7d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:43:41,010 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,010 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,010 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,010 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,011 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-13T19:43:41,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,015 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,015 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,015 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,016 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:43:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:41,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:43:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:43:41,018 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:41,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:41,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:43:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:43:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:41,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:43:41,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527081081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527081086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527081086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111339b9e748d82b43c08addf9dc5d8cff97_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527021041/Put/seqid=0 2024-11-13T19:43:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527081087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527081089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741954_1130 (size=12154) 2024-11-13T19:43:41,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:41,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111339b9e748d82b43c08addf9dc5d8cff97_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339b9e748d82b43c08addf9dc5d8cff97_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:41,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/1f5e19159a29401885fd50183d83de00, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:41,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/1f5e19159a29401885fd50183d83de00 is 175, key is test_row_0/A:col10/1731527021041/Put/seqid=0 2024-11-13T19:43:41,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741955_1131 (size=30955) 2024-11-13T19:43:41,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/1f5e19159a29401885fd50183d83de00 2024-11-13T19:43:41,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527081194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527081206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527081206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527081207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527081206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/82654f1b0bb8430ba2ddb99ffab380ae is 50, key is test_row_0/B:col10/1731527021041/Put/seqid=0 2024-11-13T19:43:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741956_1132 (size=12001) 2024-11-13T19:43:41,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:41,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527081414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527081423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527081424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527081424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527081425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:41,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/82654f1b0bb8430ba2ddb99ffab380ae 2024-11-13T19:43:41,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dd8cb47daa5646699204b8e90dcdfe2b is 50, key is test_row_0/C:col10/1731527021041/Put/seqid=0 2024-11-13T19:43:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741957_1133 (size=12001) 2024-11-13T19:43:41,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dd8cb47daa5646699204b8e90dcdfe2b 2024-11-13T19:43:41,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/1f5e19159a29401885fd50183d83de00 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00 2024-11-13T19:43:41,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T19:43:41,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/82654f1b0bb8430ba2ddb99ffab380ae as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae 2024-11-13T19:43:41,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:43:41,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dd8cb47daa5646699204b8e90dcdfe2b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b 2024-11-13T19:43:41,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:43:41,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for da41fb4884d60614ee8b9d1f2d3eb0fa in 692ms, sequenceid=16, compaction requested=false 2024-11-13T19:43:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:41,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:41,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527081762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527081778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527081786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527081788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527081793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d13718b129634203a36dbb0840b82fa1_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527021087/Put/seqid=0 2024-11-13T19:43:41,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741958_1134 (size=17034) 2024-11-13T19:43:41,843 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:41,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d13718b129634203a36dbb0840b82fa1_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d13718b129634203a36dbb0840b82fa1_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:41,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/08ade9a1508e4490bf037f75640c657a, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:41,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/08ade9a1508e4490bf037f75640c657a is 175, key is test_row_0/A:col10/1731527021087/Put/seqid=0 2024-11-13T19:43:41,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741959_1135 (size=48139) 2024-11-13T19:43:41,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=38, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/08ade9a1508e4490bf037f75640c657a 2024-11-13T19:43:41,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e538a0bdfad0433c8a7ed7a803bdabd8 is 50, key is test_row_0/B:col10/1731527021087/Put/seqid=0 2024-11-13T19:43:41,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527081894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527081904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741960_1136 (size=12001) 2024-11-13T19:43:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527081906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527081914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527081914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:41,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:41,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:41,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527082103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527082115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527082115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:42,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527082133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527082133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:42,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:42,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:42,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e538a0bdfad0433c8a7ed7a803bdabd8 2024-11-13T19:43:42,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/85aa56f996324e0e818d377914cd4fd8 is 50, key is test_row_0/C:col10/1731527021087/Put/seqid=0 2024-11-13T19:43:42,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741961_1137 (size=12001) 2024-11-13T19:43:42,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/85aa56f996324e0e818d377914cd4fd8 2024-11-13T19:43:42,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/08ade9a1508e4490bf037f75640c657a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a 2024-11-13T19:43:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527082424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527082424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527082424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:42,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a, entries=250, sequenceid=38, filesize=47.0 K 2024-11-13T19:43:42,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e538a0bdfad0433c8a7ed7a803bdabd8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8 2024-11-13T19:43:42,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527082455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527082455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:42,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T19:43:42,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/85aa56f996324e0e818d377914cd4fd8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8 2024-11-13T19:43:42,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T19:43:42,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for da41fb4884d60614ee8b9d1f2d3eb0fa in 782ms, sequenceid=38, compaction requested=false 2024-11-13T19:43:42,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:42,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T19:43:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:42,594 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:43:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:42,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:42,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c43429c6b4fe445dbff5d9acf142049e_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527021756/Put/seqid=0 2024-11-13T19:43:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741962_1138 (size=12154) 2024-11-13T19:43:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:42,692 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c43429c6b4fe445dbff5d9acf142049e_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c43429c6b4fe445dbff5d9acf142049e_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:42,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/16139d9a877a41fc8885af94b751254f, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/16139d9a877a41fc8885af94b751254f is 175, key is test_row_0/A:col10/1731527021756/Put/seqid=0 2024-11-13T19:43:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741963_1139 (size=30955) 2024-11-13T19:43:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:42,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527082994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527083000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527083007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527022985, max timestamp=1731527023010Num uniques 859; 2024-11-13T19:43:43,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:43:43,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527022985, max timestamp=1731527023010Num uniques 859; 2024-11-13T19:43:43,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:43:43,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527022985, max timestamp=1731527023010Num uniques 859; 2024-11-13T19:43:43,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:43:43,136 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/16139d9a877a41fc8885af94b751254f 2024-11-13T19:43:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527083123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527083124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527083124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527083131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527083144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/9185adeafe6e4de0a466541eb5356a53 is 50, key is test_row_0/B:col10/1731527021756/Put/seqid=0 2024-11-13T19:43:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:43,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741964_1140 (size=12001) 2024-11-13T19:43:43,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527083256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527083256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527083353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527083354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527083354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527083474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527083474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,588 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/9185adeafe6e4de0a466541eb5356a53 2024-11-13T19:43:43,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1cf74f61e1404f38aaf39e454c72098f is 50, key is test_row_0/C:col10/1731527021756/Put/seqid=0 2024-11-13T19:43:43,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741965_1141 (size=12001) 2024-11-13T19:43:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527083684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527083684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527083689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527083794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527083794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:43,918 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:43:44,030 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1cf74f61e1404f38aaf39e454c72098f 2024-11-13T19:43:44,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/16139d9a877a41fc8885af94b751254f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f 2024-11-13T19:43:44,064 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T19:43:44,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/9185adeafe6e4de0a466541eb5356a53 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53 2024-11-13T19:43:44,085 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T19:43:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1cf74f61e1404f38aaf39e454c72098f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f 2024-11-13T19:43:44,100 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T19:43:44,103 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1509ms, sequenceid=49, compaction requested=true 2024-11-13T19:43:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T19:43:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T19:43:44,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T19:43:44,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0870 sec 2024-11-13T19:43:44,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 3.1030 sec 2024-11-13T19:43:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:44,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-13T19:43:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:44,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1885e5f848d4db7a979d02035b0f0b7_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527084254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527084257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527084258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741966_1142 (size=17034) 2024-11-13T19:43:44,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:44,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1885e5f848d4db7a979d02035b0f0b7_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1885e5f848d4db7a979d02035b0f0b7_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:44,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/f4a5ad5b62664a638577c681b59f1f71, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:44,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/f4a5ad5b62664a638577c681b59f1f71 is 175, key is test_row_0/A:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527084314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527084314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741967_1143 (size=48139) 2024-11-13T19:43:44,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/f4a5ad5b62664a638577c681b59f1f71 2024-11-13T19:43:44,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c63791c10575474fa52824b2878c0a15 is 50, key is test_row_0/B:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527084375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527084375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527084377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741968_1144 (size=12001) 2024-11-13T19:43:44,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c63791c10575474fa52824b2878c0a15 2024-11-13T19:43:44,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/71f7144dcd034fbb87df7495d9889c76 is 50, key is test_row_0/C:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741969_1145 (size=12001) 2024-11-13T19:43:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527084585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527084594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527084594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:44,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/71f7144dcd034fbb87df7495d9889c76 2024-11-13T19:43:44,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/f4a5ad5b62664a638577c681b59f1f71 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71 2024-11-13T19:43:44,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71, entries=250, sequenceid=85, filesize=47.0 K 2024-11-13T19:43:44,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c63791c10575474fa52824b2878c0a15 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15 2024-11-13T19:43:44,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T19:43:44,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/71f7144dcd034fbb87df7495d9889c76 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76 2024-11-13T19:43:44,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T19:43:44,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for da41fb4884d60614ee8b9d1f2d3eb0fa in 645ms, sequenceid=85, compaction requested=true 2024-11-13T19:43:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:44,859 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:44,859 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:44,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:44,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:44,865 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:44,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:44,866 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:44,866 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:44,866 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:44,866 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:44,866 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=46.9 K 2024-11-13T19:43:44,866 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=154.5 K 2024-11-13T19:43:44,866 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:44,866 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 82654f1b0bb8430ba2ddb99ffab380ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527021031 2024-11-13T19:43:44,866 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71] 2024-11-13T19:43:44,867 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e538a0bdfad0433c8a7ed7a803bdabd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731527021081 2024-11-13T19:43:44,867 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5e19159a29401885fd50183d83de00, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527021031 2024-11-13T19:43:44,868 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9185adeafe6e4de0a466541eb5356a53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527021756 2024-11-13T19:43:44,868 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ade9a1508e4490bf037f75640c657a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731527021071 2024-11-13T19:43:44,868 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c63791c10575474fa52824b2878c0a15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:44,869 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16139d9a877a41fc8885af94b751254f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527021756 2024-11-13T19:43:44,869 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4a5ad5b62664a638577c681b59f1f71, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:44,885 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:44,886 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#126 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:44,887 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/6e3404fba3cd490297b87d1e7188b78b is 50, key is test_row_0/B:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,894 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139c04af5327fb4981baa4a42381828c1f_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:44,899 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139c04af5327fb4981baa4a42381828c1f_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:44,900 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c04af5327fb4981baa4a42381828c1f_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:44,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:44,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741970_1146 (size=12139) 2024-11-13T19:43:44,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ad58def32b74a32a0cfda223d1a7842_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527024249/Put/seqid=0 2024-11-13T19:43:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741971_1147 (size=4469) 2024-11-13T19:43:44,967 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#127 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:44,968 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/151ee9fe6dd74f88a43ab03c4f2489b6 is 175, key is test_row_0/A:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:44,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527024944, max timestamp=1731527024977Num uniques 859; 2024-11-13T19:43:44,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:44,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527024944, max timestamp=1731527024977Num uniques 859; 2024-11-13T19:43:44,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:44,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527024944, max timestamp=1731527024977Num uniques 859; 2024-11-13T19:43:44,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:44,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741972_1148 (size=14594) 2024-11-13T19:43:45,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741973_1149 (size=31093) 2024-11-13T19:43:45,020 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/151ee9fe6dd74f88a43ab03c4f2489b6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6 2024-11-13T19:43:45,031 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into 151ee9fe6dd74f88a43ab03c4f2489b6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:45,031 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:45,031 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=12, startTime=1731527024859; duration=0sec 2024-11-13T19:43:45,031 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:45,031 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:45,032 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:45,035 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:45,035 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:45,035 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:45,035 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=46.9 K 2024-11-13T19:43:45,036 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8cb47daa5646699204b8e90dcdfe2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527021031 2024-11-13T19:43:45,036 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85aa56f996324e0e818d377914cd4fd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731527021081 2024-11-13T19:43:45,037 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf74f61e1404f38aaf39e454c72098f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527021756 2024-11-13T19:43:45,038 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f7144dcd034fbb87df7495d9889c76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:45,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527085041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527085042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527085044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,081 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:45,082 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/8d8644e0355a4696b8b57e5d3149782f is 50, key is test_row_0/C:col10/1731527024211/Put/seqid=0 2024-11-13T19:43:45,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741974_1150 (size=12139) 2024-11-13T19:43:45,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527085155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527085156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527085165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T19:43:45,207 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:45,207 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-13T19:43:45,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:45,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:45,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:45,328 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/6e3404fba3cd490297b87d1e7188b78b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6e3404fba3cd490297b87d1e7188b78b 2024-11-13T19:43:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527085335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,335 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 6e3404fba3cd490297b87d1e7188b78b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:45,335 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:45,335 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=12, startTime=1731527024859; duration=0sec 2024-11-13T19:43:45,336 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:45,336 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527085335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,336 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,337 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T19:43:45,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:45,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:45,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:45,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:45,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527085373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527085373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527085384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:45,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ad58def32b74a32a0cfda223d1a7842_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ad58def32b74a32a0cfda223d1a7842_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:45,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/025e272dd48c4202b0800f81ec7201f2, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:45,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/025e272dd48c4202b0800f81ec7201f2 is 175, key is test_row_0/A:col10/1731527024249/Put/seqid=0 2024-11-13T19:43:45,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741975_1151 (size=39549) 2024-11-13T19:43:45,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/025e272dd48c4202b0800f81ec7201f2 2024-11-13T19:43:45,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c7926540bcca48ab9dd5a0317765bd87 is 50, key is test_row_0/B:col10/1731527024249/Put/seqid=0 2024-11-13T19:43:45,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741976_1152 (size=12001) 2024-11-13T19:43:45,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c7926540bcca48ab9dd5a0317765bd87 2024-11-13T19:43:45,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/643c0cca20f04437978063f997996c75 is 50, key is test_row_0/C:col10/1731527024249/Put/seqid=0 2024-11-13T19:43:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741977_1153 (size=12001) 2024-11-13T19:43:45,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/643c0cca20f04437978063f997996c75 2024-11-13T19:43:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/025e272dd48c4202b0800f81ec7201f2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2 2024-11-13T19:43:45,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2, entries=200, sequenceid=95, filesize=38.6 K 2024-11-13T19:43:45,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c7926540bcca48ab9dd5a0317765bd87 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87 2024-11-13T19:43:45,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T19:43:45,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/643c0cca20f04437978063f997996c75 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75 2024-11-13T19:43:45,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T19:43:45,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for da41fb4884d60614ee8b9d1f2d3eb0fa in 594ms, sequenceid=95, compaction requested=false 2024-11-13T19:43:45,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:45,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T19:43:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:45,522 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:45,533 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/8d8644e0355a4696b8b57e5d3149782f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/8d8644e0355a4696b8b57e5d3149782f 2024-11-13T19:43:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113639bd22130504461a336fca7570f2cbe_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527025037/Put/seqid=0 2024-11-13T19:43:45,544 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into 8d8644e0355a4696b8b57e5d3149782f(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:45,544 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:45,544 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=12, startTime=1731527024859; duration=0sec 2024-11-13T19:43:45,544 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:45,544 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:45,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741978_1154 (size=12204) 2024-11-13T19:43:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527085706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527085707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527085708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527085813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527085814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527085815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:45,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:45,966 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113639bd22130504461a336fca7570f2cbe_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113639bd22130504461a336fca7570f2cbe_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:45,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/dca623b62f6d48768a7fb6ded61669ab, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/dca623b62f6d48768a7fb6ded61669ab is 175, key is test_row_0/A:col10/1731527025037/Put/seqid=0 2024-11-13T19:43:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741979_1155 (size=31005) 2024-11-13T19:43:45,984 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/dca623b62f6d48768a7fb6ded61669ab 2024-11-13T19:43:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/93de25d99cc2408fbee408ace544e8a8 is 50, key is test_row_0/B:col10/1731527025037/Put/seqid=0 2024-11-13T19:43:46,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741980_1156 (size=12051) 2024-11-13T19:43:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527086025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527086025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527086025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527086337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527086344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527086344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:46,418 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/93de25d99cc2408fbee408ace544e8a8 2024-11-13T19:43:46,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dce3b0663d26495e8e69c495e31be1ee is 50, key is test_row_0/C:col10/1731527025037/Put/seqid=0 2024-11-13T19:43:46,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741981_1157 (size=12051) 2024-11-13T19:43:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527086854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527086854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527086866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:46,921 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dce3b0663d26495e8e69c495e31be1ee 2024-11-13T19:43:46,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/dca623b62f6d48768a7fb6ded61669ab as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab 2024-11-13T19:43:46,949 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab, entries=150, sequenceid=132, filesize=30.3 K 2024-11-13T19:43:46,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/93de25d99cc2408fbee408ace544e8a8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8 2024-11-13T19:43:46,956 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T19:43:46,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/dce3b0663d26495e8e69c495e31be1ee as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee 2024-11-13T19:43:46,980 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T19:43:46,982 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1460ms, sequenceid=132, compaction requested=true 2024-11-13T19:43:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T19:43:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T19:43:46,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T19:43:46,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7780 sec 2024-11-13T19:43:46,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.7850 sec 2024-11-13T19:43:47,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:43:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:47,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:43:47,366 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:47,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T19:43:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:47,373 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:47,374 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:47,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:47,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dee51ae40906456bb59b8c37a2b8d280_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:47,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741982_1158 (size=12304) 2024-11-13T19:43:47,421 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:47,431 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dee51ae40906456bb59b8c37a2b8d280_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dee51ae40906456bb59b8c37a2b8d280_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:47,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/17527cb5d50f4272bdd61460a1e91c27, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:47,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/17527cb5d50f4272bdd61460a1e91c27 is 175, key is test_row_0/A:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:47,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731527027419, max timestamp=1731527027453Num uniques 859; 2024-11-13T19:43:47,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:43:47,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731527027419, max timestamp=1731527027453Num uniques 859; 2024-11-13T19:43:47,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:47,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731527027419, max timestamp=1731527027453Num uniques 859; 2024-11-13T19:43:47,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:43:47,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741983_1159 (size=31105) 2024-11-13T19:43:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:47,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527087542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527087542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527087654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527087664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:47,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:47,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/17527cb5d50f4272bdd61460a1e91c27 2024-11-13T19:43:47,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527087865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527087875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,885 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bc86baef6a614542927483e05105334e is 50, key is test_row_0/B:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:47,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527087890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,897 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527087890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527087893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,899 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:47,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741984_1160 (size=12151) 2024-11-13T19:43:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:47,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:47,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527088186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:48,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527088206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:48,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:48,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bc86baef6a614542927483e05105334e 2024-11-13T19:43:48,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/6b953ae445c84ff79a38d3328df80eb5 is 50, key is test_row_0/C:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:48,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741985_1161 (size=12151) 2024-11-13T19:43:48,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:48,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527088705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:48,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527088724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:48,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/6b953ae445c84ff79a38d3328df80eb5 2024-11-13T19:43:48,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/17527cb5d50f4272bdd61460a1e91c27 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27 2024-11-13T19:43:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:48,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27, entries=150, sequenceid=145, filesize=30.4 K 2024-11-13T19:43:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bc86baef6a614542927483e05105334e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e 2024-11-13T19:43:48,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T19:43:48,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/6b953ae445c84ff79a38d3328df80eb5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5 2024-11-13T19:43:48,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T19:43:48,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=131.94 KB/135110 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1462ms, sequenceid=145, compaction requested=true 2024-11-13T19:43:48,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:48,824 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:48,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:48,826 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:48,828 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:48,828 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:48,828 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,828 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=129.6 K 2024-11-13T19:43:48,828 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,828 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27] 2024-11-13T19:43:48,829 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:48,829 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:48,829 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,829 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6e3404fba3cd490297b87d1e7188b78b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=47.2 K 2024-11-13T19:43:48,830 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 151ee9fe6dd74f88a43ab03c4f2489b6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:48,830 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3404fba3cd490297b87d1e7188b78b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:48,830 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 025e272dd48c4202b0800f81ec7201f2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527024218 2024-11-13T19:43:48,831 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c7926540bcca48ab9dd5a0317765bd87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527024249 2024-11-13T19:43:48,831 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca623b62f6d48768a7fb6ded61669ab, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527025023 2024-11-13T19:43:48,832 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 93de25d99cc2408fbee408ace544e8a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527025023 2024-11-13T19:43:48,832 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bc86baef6a614542927483e05105334e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:48,861 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17527cb5d50f4272bdd61460a1e91c27, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:48,886 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:48,886 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/84ec4ff4eefd4c329077bdf459cda778 is 50, key is test_row_0/B:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:48,890 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:48,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741986_1162 (size=12425) 2024-11-13T19:43:48,922 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113776b78ebbc5144f08e1324ac0a627162_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:48,925 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/84ec4ff4eefd4c329077bdf459cda778 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/84ec4ff4eefd4c329077bdf459cda778 2024-11-13T19:43:48,927 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113776b78ebbc5144f08e1324ac0a627162_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:48,927 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113776b78ebbc5144f08e1324ac0a627162_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:48,933 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 84ec4ff4eefd4c329077bdf459cda778(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:48,933 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:48,933 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=12, startTime=1731527028826; duration=0sec 2024-11-13T19:43:48,934 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:48,934 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:48,934 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:48,936 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:48,936 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:48,936 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,936 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/8d8644e0355a4696b8b57e5d3149782f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=47.2 K 2024-11-13T19:43:48,937 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8644e0355a4696b8b57e5d3149782f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731527023078 2024-11-13T19:43:48,937 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 643c0cca20f04437978063f997996c75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527024249 2024-11-13T19:43:48,938 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting dce3b0663d26495e8e69c495e31be1ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527025023 2024-11-13T19:43:48,939 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b953ae445c84ff79a38d3328df80eb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:48,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:48,946 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=131.94 KB heapSize=370.53 KB 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741987_1163 (size=4469) 2024-11-13T19:43:48,957 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#139 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:48,957 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:48,958 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/54f3cf16ec2b4e60b41f2aa88174cbb0 is 175, key is test_row_0/A:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:48,958 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/a609750091ce421ca3919b30493c5be1 is 50, key is test_row_0/C:col10/1731527025690/Put/seqid=0 2024-11-13T19:43:48,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137c1998a8247c41b2a4e31f92392a1b25_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527027539/Put/seqid=0 2024-11-13T19:43:48,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741988_1164 (size=31379) 2024-11-13T19:43:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741989_1165 (size=12425) 2024-11-13T19:43:48,991 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/a609750091ce421ca3919b30493c5be1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/a609750091ce421ca3919b30493c5be1 2024-11-13T19:43:48,991 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/54f3cf16ec2b4e60b41f2aa88174cbb0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0 2024-11-13T19:43:49,001 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into a609750091ce421ca3919b30493c5be1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:49,001 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:49,001 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=12, startTime=1731527028826; duration=0sec 2024-11-13T19:43:49,001 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:49,001 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:49,003 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into 54f3cf16ec2b4e60b41f2aa88174cbb0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:49,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:49,003 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=12, startTime=1731527028824; duration=0sec 2024-11-13T19:43:49,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:49,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741990_1166 (size=12304) 2024-11-13T19:43:49,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,013 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137c1998a8247c41b2a4e31f92392a1b25_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c1998a8247c41b2a4e31f92392a1b25_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a7967be256974044a6c81fc5553be3aa, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a7967be256974044a6c81fc5553be3aa is 175, key is test_row_0/A:col10/1731527027539/Put/seqid=0 2024-11-13T19:43:49,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741991_1167 (size=31105) 2024-11-13T19:43:49,037 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a7967be256974044a6c81fc5553be3aa 2024-11-13T19:43:49,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/647d8b4886c741f2a270cde580104dd4 is 50, key is test_row_0/B:col10/1731527027539/Put/seqid=0 2024-11-13T19:43:49,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741992_1168 (size=12151) 2024-11-13T19:43:49,328 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:43:49,328 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T19:43:49,486 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/647d8b4886c741f2a270cde580104dd4 2024-11-13T19:43:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/432d01efe50c4b9889616ceaea2b8405 is 50, key is test_row_0/C:col10/1731527027539/Put/seqid=0 2024-11-13T19:43:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741993_1169 (size=12151) 2024-11-13T19:43:49,549 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/432d01efe50c4b9889616ceaea2b8405 2024-11-13T19:43:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:49,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a7967be256974044a6c81fc5553be3aa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa 2024-11-13T19:43:49,560 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa, entries=150, sequenceid=181, filesize=30.4 K 2024-11-13T19:43:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/647d8b4886c741f2a270cde580104dd4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4 2024-11-13T19:43:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,566 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T19:43:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/432d01efe50c4b9889616ceaea2b8405 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405 2024-11-13T19:43:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,581 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T19:43:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,586 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~370.48 KB/379376, currentSize=0 B/0 for da41fb4884d60614ee8b9d1f2d3eb0fa in 639ms, sequenceid=181, compaction requested=false 2024-11-13T19:43:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T19:43:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T19:43:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T19:43:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2340 sec 2024-11-13T19:43:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.2470 sec 2024-11-13T19:43:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:49,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c16d33504bd545e68c3b797aa39d2134_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741994_1170 (size=19774) 2024-11-13T19:43:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731527029903, max timestamp=1731527029945Num uniques 859; 2024-11-13T19:43:49,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-13T19:43:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731527029903, max timestamp=1731527029945Num uniques 859; 2024-11-13T19:43:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-13T19:43:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731527029903, max timestamp=1731527029945Num uniques 859; 2024-11-13T19:43:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-13T19:43:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527089962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527089964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527089965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527089966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527089967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527090073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527090084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527090084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527090084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527090084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c16d33504bd545e68c3b797aa39d2134_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c16d33504bd545e68c3b797aa39d2134_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:50,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/bb0022ddd9c44db79d84a6165f689f57, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/bb0022ddd9c44db79d84a6165f689f57 is 175, key is test_row_0/A:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741995_1171 (size=57029) 2024-11-13T19:43:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527090283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/bb0022ddd9c44db79d84a6165f689f57 2024-11-13T19:43:50,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527090295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527090296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527090297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527090297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c1dabb92fc1d4a35aa50b31ff8f4c37f is 50, key is test_row_0/B:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741996_1172 (size=9757) 2024-11-13T19:43:50,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c1dabb92fc1d4a35aa50b31ff8f4c37f 2024-11-13T19:43:50,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b5cf509e6e884c16ad8861b85dd63f45 is 50, key is test_row_0/C:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741997_1173 (size=9757) 2024-11-13T19:43:50,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b5cf509e6e884c16ad8861b85dd63f45 2024-11-13T19:43:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/bb0022ddd9c44db79d84a6165f689f57 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57 2024-11-13T19:43:50,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57, entries=300, sequenceid=193, filesize=55.7 K 2024-11-13T19:43:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c1dabb92fc1d4a35aa50b31ff8f4c37f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f 2024-11-13T19:43:50,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f, entries=100, sequenceid=193, filesize=9.5 K 2024-11-13T19:43:50,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b5cf509e6e884c16ad8861b85dd63f45 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45 2024-11-13T19:43:50,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45, entries=100, sequenceid=193, filesize=9.5 K 2024-11-13T19:43:50,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for da41fb4884d60614ee8b9d1f2d3eb0fa in 535ms, sequenceid=193, compaction requested=true 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:50,383 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:50,383 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:50,384 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119513 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:50,384 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:50,385 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:50,385 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:50,385 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:50,385 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/84ec4ff4eefd4c329077bdf459cda778, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=33.5 K 2024-11-13T19:43:50,385 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:50,385 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=116.7 K 2024-11-13T19:43:50,385 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:50,385 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57] 2024-11-13T19:43:50,385 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ec4ff4eefd4c329077bdf459cda778, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:50,386 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647d8b4886c741f2a270cde580104dd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731527027503 2024-11-13T19:43:50,386 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f3cf16ec2b4e60b41f2aa88174cbb0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:50,386 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1dabb92fc1d4a35aa50b31ff8f4c37f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527029814 2024-11-13T19:43:50,387 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a7967be256974044a6c81fc5553be3aa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731527027503 2024-11-13T19:43:50,387 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0022ddd9c44db79d84a6165f689f57, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527029764 2024-11-13T19:43:50,398 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:50,399 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/6df7fe1bd4b84a92af1d0fa365b24330 is 50, key is test_row_0/B:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,406 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,414 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135298d6c7fee5449c83cc58479ca82614_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,416 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135298d6c7fee5449c83cc58479ca82614_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,416 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135298d6c7fee5449c83cc58479ca82614_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741998_1174 (size=12527) 2024-11-13T19:43:50,438 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/6df7fe1bd4b84a92af1d0fa365b24330 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6df7fe1bd4b84a92af1d0fa365b24330 2024-11-13T19:43:50,445 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 6df7fe1bd4b84a92af1d0fa365b24330(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:50,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:50,446 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=13, startTime=1731527030383; duration=0sec 2024-11-13T19:43:50,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:50,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:50,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:50,447 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:50,447 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:50,447 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:50,448 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/a609750091ce421ca3919b30493c5be1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=33.5 K 2024-11-13T19:43:50,448 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a609750091ce421ca3919b30493c5be1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731527025690 2024-11-13T19:43:50,449 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 432d01efe50c4b9889616ceaea2b8405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731527027503 2024-11-13T19:43:50,450 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5cf509e6e884c16ad8861b85dd63f45, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527029814 2024-11-13T19:43:50,460 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:50,460 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/7d30459e3f8e4e789ab10c53dc642701 is 50, key is test_row_0/C:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741999_1175 (size=4469) 2024-11-13T19:43:50,463 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#148 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:50,464 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b535dbf27bfa4fd09e5ba278248b6bb5 is 175, key is test_row_0/A:col10/1731527029814/Put/seqid=0 2024-11-13T19:43:50,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742001_1177 (size=31588) 2024-11-13T19:43:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742000_1176 (size=12527) 2024-11-13T19:43:50,529 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/7d30459e3f8e4e789ab10c53dc642701 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/7d30459e3f8e4e789ab10c53dc642701 2024-11-13T19:43:50,535 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into 7d30459e3f8e4e789ab10c53dc642701(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:50,535 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:50,535 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=13, startTime=1731527030383; duration=0sec 2024-11-13T19:43:50,535 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:50,535 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:50,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=161.02 KB heapSize=422.48 KB 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527090614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527090615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527090620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527090621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e772c96c936a42b6823dc6cded941ac6_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527029963/Put/seqid=0 2024-11-13T19:43:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527090622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742002_1178 (size=14794) 2024-11-13T19:43:50,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e772c96c936a42b6823dc6cded941ac6_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e772c96c936a42b6823dc6cded941ac6_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:50,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43e1e85336424e9096794267b074d77f, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:50,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43e1e85336424e9096794267b074d77f is 175, key is test_row_0/A:col10/1731527029963/Put/seqid=0 2024-11-13T19:43:50,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742003_1179 (size=39749) 2024-11-13T19:43:50,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43e1e85336424e9096794267b074d77f 2024-11-13T19:43:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527090724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527090734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527090735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527090746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:50,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e7287dde078445b2b3ccb1b02f7b7032 is 50, key is test_row_0/B:col10/1731527029963/Put/seqid=0 2024-11-13T19:43:50,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742004_1180 (size=12151) 2024-11-13T19:43:50,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e7287dde078445b2b3ccb1b02f7b7032 2024-11-13T19:43:50,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d8ae9736f4dd42949f3b7e70d1fa1965 is 50, key is test_row_0/C:col10/1731527029963/Put/seqid=0 2024-11-13T19:43:50,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742005_1181 (size=12151) 2024-11-13T19:43:50,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d8ae9736f4dd42949f3b7e70d1fa1965 2024-11-13T19:43:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43e1e85336424e9096794267b074d77f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f 2024-11-13T19:43:50,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f, entries=200, sequenceid=227, filesize=38.8 K 2024-11-13T19:43:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/e7287dde078445b2b3ccb1b02f7b7032 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032 2024-11-13T19:43:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032, entries=150, sequenceid=227, filesize=11.9 K 2024-11-13T19:43:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d8ae9736f4dd42949f3b7e70d1fa1965 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965 2024-11-13T19:43:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965, entries=150, sequenceid=227, filesize=11.9 K 2024-11-13T19:43:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.44 KB/432576, currentSize=33.54 KB/34350 for da41fb4884d60614ee8b9d1f2d3eb0fa in 304ms, sequenceid=227, compaction requested=false 2024-11-13T19:43:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,944 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b535dbf27bfa4fd09e5ba278248b6bb5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5 2024-11-13T19:43:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,953 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into b535dbf27bfa4fd09e5ba278248b6bb5(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:50,953 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,953 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=13, startTime=1731527030383; duration=0sec 2024-11-13T19:43:50,953 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:50,953 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:50,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:43:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:50,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d79af1e36ad346b38ac2af79d4aa7e08_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:51,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527091009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527091010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527091011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527091013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742006_1182 (size=19774) 2024-11-13T19:43:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527091127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527091127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527091134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527091134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527091134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527091334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527091344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527091344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527091344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:51,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d79af1e36ad346b38ac2af79d4aa7e08_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d79af1e36ad346b38ac2af79d4aa7e08_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:51,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6ffd3dfd08254e4385ee88d50adfb017, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:51,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6ffd3dfd08254e4385ee88d50adfb017 is 175, key is test_row_0/A:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:51,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742007_1183 (size=57033) 2024-11-13T19:43:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T19:43:51,564 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:51,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T19:43:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:51,567 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:51,568 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:51,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527091643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527091654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527091654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527091663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:51,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:51,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:51,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:51,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:51,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:51,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:51,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6ffd3dfd08254e4385ee88d50adfb017 2024-11-13T19:43:51,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/4c96f64dff7b4d4b8876774af8622c8a is 50, key is test_row_0/B:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:51,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742008_1184 (size=12151) 2024-11-13T19:43:51,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/4c96f64dff7b4d4b8876774af8622c8a 2024-11-13T19:43:51,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/0c03b7bb99584ed8b2357f4a9b00fe59 is 50, key is test_row_0/C:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742009_1185 (size=12151) 2024-11-13T19:43:52,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527092143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:52,150 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:52,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527092164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527092164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527092164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:52,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527092184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:52,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:52,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:52,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:52,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:52,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/0c03b7bb99584ed8b2357f4a9b00fe59 2024-11-13T19:43:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6ffd3dfd08254e4385ee88d50adfb017 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017 2024-11-13T19:43:52,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017, entries=300, sequenceid=240, filesize=55.7 K 2024-11-13T19:43:52,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/4c96f64dff7b4d4b8876774af8622c8a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a 2024-11-13T19:43:52,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T19:43:52,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/0c03b7bb99584ed8b2357f4a9b00fe59 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59 2024-11-13T19:43:52,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T19:43:52,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1441ms, sequenceid=240, compaction requested=true 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:52,404 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:52,404 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:52,405 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128370 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:52,405 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:52,406 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,406 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=125.4 K 2024-11-13T19:43:52,406 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,406 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017] 2024-11-13T19:43:52,406 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b535dbf27bfa4fd09e5ba278248b6bb5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527027510 2024-11-13T19:43:52,407 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e1e85336424e9096794267b074d77f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731527029963 2024-11-13T19:43:52,407 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ffd3dfd08254e4385ee88d50adfb017, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030615 2024-11-13T19:43:52,409 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:52,409 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:52,409 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,409 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6df7fe1bd4b84a92af1d0fa365b24330, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.0 K 2024-11-13T19:43:52,410 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df7fe1bd4b84a92af1d0fa365b24330, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527027510 2024-11-13T19:43:52,411 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e7287dde078445b2b3ccb1b02f7b7032, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731527029963 2024-11-13T19:43:52,412 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c96f64dff7b4d4b8876774af8622c8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030960 2024-11-13T19:43:52,434 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:52,444 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:52,444 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/96e3d62eea024c8db7f6fb372a61e66a is 50, key is test_row_0/B:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:52,446 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411130be76fd33d6c40adafc1a4dd5cff54d9_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:52,448 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411130be76fd33d6c40adafc1a4dd5cff54d9_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:52,448 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130be76fd33d6c40adafc1a4dd5cff54d9_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:52,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T19:43:52,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,498 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:52,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742010_1186 (size=12629) 2024-11-13T19:43:52,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742011_1187 (size=4469) 2024-11-13T19:43:52,512 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#156 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:52,513 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/fb78ed248779448ba3505d071cc440d3 is 175, key is test_row_0/A:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:52,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130814f09867e34fd5a9cf2ecd0994eba2_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527030997/Put/seqid=0 2024-11-13T19:43:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742012_1188 (size=31583) 2024-11-13T19:43:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742013_1189 (size=12404) 2024-11-13T19:43:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:52,608 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130814f09867e34fd5a9cf2ecd0994eba2_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130814f09867e34fd5a9cf2ecd0994eba2_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:52,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3553aada894b5faeaeb25c259c8ff5, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3553aada894b5faeaeb25c259c8ff5 is 175, key is test_row_0/A:col10/1731527030997/Put/seqid=0 2024-11-13T19:43:52,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742014_1190 (size=31205) 2024-11-13T19:43:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:52,931 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/96e3d62eea024c8db7f6fb372a61e66a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/96e3d62eea024c8db7f6fb372a61e66a 2024-11-13T19:43:52,939 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 96e3d62eea024c8db7f6fb372a61e66a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:52,939 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:52,939 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=13, startTime=1731527032404; duration=0sec 2024-11-13T19:43:52,940 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:52,940 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:52,940 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:52,942 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:52,943 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:52,943 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:52,943 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/7d30459e3f8e4e789ab10c53dc642701, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.0 K 2024-11-13T19:43:52,944 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d30459e3f8e4e789ab10c53dc642701, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731527027510 2024-11-13T19:43:52,945 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ae9736f4dd42949f3b7e70d1fa1965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731527029963 2024-11-13T19:43:52,945 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c03b7bb99584ed8b2357f4a9b00fe59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030960 2024-11-13T19:43:52,967 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:52,968 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/68140f81cc434202878e03e2a02eeeac is 50, key is test_row_0/C:col10/1731527030961/Put/seqid=0 2024-11-13T19:43:53,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742015_1191 (size=12629) 2024-11-13T19:43:53,017 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/fb78ed248779448ba3505d071cc440d3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3 2024-11-13T19:43:53,030 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3553aada894b5faeaeb25c259c8ff5 2024-11-13T19:43:53,032 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into fb78ed248779448ba3505d071cc440d3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:53,032 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:53,032 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=13, startTime=1731527032404; duration=0sec 2024-11-13T19:43:53,033 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:53,033 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/2bc6b8c763ee4f40a966038e7507691a is 50, key is test_row_0/B:col10/1731527030997/Put/seqid=0 2024-11-13T19:43:53,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742016_1192 (size=12251) 2024-11-13T19:43:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527093200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527093203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527093206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527093208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527093318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527093327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527093329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527093333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,430 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/68140f81cc434202878e03e2a02eeeac as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/68140f81cc434202878e03e2a02eeeac 2024-11-13T19:43:53,443 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into 68140f81cc434202878e03e2a02eeeac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:53,443 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:53,443 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=13, startTime=1731527032404; duration=0sec 2024-11-13T19:43:53,444 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:53,444 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:53,474 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/2bc6b8c763ee4f40a966038e7507691a 2024-11-13T19:43:53,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f07ead95d74143baa766a89d24847e7c is 50, key is test_row_0/C:col10/1731527030997/Put/seqid=0 2024-11-13T19:43:53,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527093540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527093545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527093545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527093544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742017_1193 (size=12251) 2024-11-13T19:43:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527093853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527093853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527093854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527093864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:53,955 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f07ead95d74143baa766a89d24847e7c 2024-11-13T19:43:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3553aada894b5faeaeb25c259c8ff5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5 2024-11-13T19:43:54,000 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5, entries=150, sequenceid=260, filesize=30.5 K 2024-11-13T19:43:54,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/2bc6b8c763ee4f40a966038e7507691a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a 2024-11-13T19:43:54,012 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T19:43:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f07ead95d74143baa766a89d24847e7c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c 2024-11-13T19:43:54,035 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c, entries=150, sequenceid=260, filesize=12.0 K 2024-11-13T19:43:54,036 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1538ms, sequenceid=260, compaction requested=false 2024-11-13T19:43:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T19:43:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T19:43:54,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T19:43:54,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-13T19:43:54,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.4790 sec 2024-11-13T19:43:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:54,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:43:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:54,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:54,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139aecb96f0237430cb65f89eb1c07ac1e_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:54,198 INFO [master/54d0784d1943:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T19:43:54,198 INFO [master/54d0784d1943:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T19:43:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742018_1194 (size=14994) 2024-11-13T19:43:54,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527094280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527094363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527094363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527094363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527094374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527094393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527094606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:54,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:54,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139aecb96f0237430cb65f89eb1c07ac1e_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aecb96f0237430cb65f89eb1c07ac1e_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:54,642 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3312351bdf48efa01cc5a8a7a80d99, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:54,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3312351bdf48efa01cc5a8a7a80d99 is 175, key is test_row_0/A:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742019_1195 (size=39949) 2024-11-13T19:43:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527094917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3312351bdf48efa01cc5a8a7a80d99 2024-11-13T19:43:55,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f2732878f9b14c39899fd866a6341fd7 is 50, key is test_row_0/B:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:55,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742020_1196 (size=12301) 2024-11-13T19:43:55,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f2732878f9b14c39899fd866a6341fd7 2024-11-13T19:43:55,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d9af16aa56b84ea293e68ccda39cec6d is 50, key is test_row_0/C:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:55,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742021_1197 (size=12301) 2024-11-13T19:43:55,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d9af16aa56b84ea293e68ccda39cec6d 2024-11-13T19:43:55,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/ee3312351bdf48efa01cc5a8a7a80d99 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99 2024-11-13T19:43:55,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99, entries=200, sequenceid=276, filesize=39.0 K 2024-11-13T19:43:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f2732878f9b14c39899fd866a6341fd7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7 2024-11-13T19:43:55,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T19:43:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/d9af16aa56b84ea293e68ccda39cec6d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d 2024-11-13T19:43:55,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T19:43:55,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1163ms, sequenceid=276, compaction requested=true 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:43:55,340 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:55,341 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:55,349 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:55,349 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:55,349 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,349 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/68140f81cc434202878e03e2a02eeeac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.3 K 2024-11-13T19:43:55,353 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:55,353 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:55,353 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,354 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=100.3 K 2024-11-13T19:43:55,354 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,354 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99] 2024-11-13T19:43:55,354 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 68140f81cc434202878e03e2a02eeeac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030960 2024-11-13T19:43:55,355 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb78ed248779448ba3505d071cc440d3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030960 2024-11-13T19:43:55,356 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f07ead95d74143baa766a89d24847e7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731527030997 2024-11-13T19:43:55,356 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee3553aada894b5faeaeb25c259c8ff5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731527030997 2024-11-13T19:43:55,358 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d9af16aa56b84ea293e68ccda39cec6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033204 2024-11-13T19:43:55,359 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee3312351bdf48efa01cc5a8a7a80d99, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033197 2024-11-13T19:43:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:43:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:55,420 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:55,421 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/ddec1ec315664d71bb7788a6d9a61472 is 50, key is test_row_0/C:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:55,424 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527095418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527095421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527095425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527095426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527095434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac1e91cfd0a94b4aaf55c8dacb172e87_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527034267/Put/seqid=0 2024-11-13T19:43:55,474 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113867cee4486fa428cbb1f9c7c09f2db42_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:55,476 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113867cee4486fa428cbb1f9c7c09f2db42_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:55,477 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113867cee4486fa428cbb1f9c7c09f2db42_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:55,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742022_1198 (size=12881) 2024-11-13T19:43:55,526 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/ddec1ec315664d71bb7788a6d9a61472 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/ddec1ec315664d71bb7788a6d9a61472 2024-11-13T19:43:55,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527095535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,544 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into ddec1ec315664d71bb7788a6d9a61472(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:55,544 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:55,544 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=13, startTime=1731527035340; duration=0sec 2024-11-13T19:43:55,544 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:55,544 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:55,544 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:43:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527095545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742024_1200 (size=4469) 2024-11-13T19:43:55,552 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#166 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:55,553 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/2a665e137c1c464a8cea224f08f68e18 is 175, key is test_row_0/A:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,554 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:43:55,560 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:55,560 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,560 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/96e3d62eea024c8db7f6fb372a61e66a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.3 K 2024-11-13T19:43:55,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e3d62eea024c8db7f6fb372a61e66a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731527030960 2024-11-13T19:43:55,563 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc6b8c763ee4f40a966038e7507691a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731527030997 2024-11-13T19:43:55,564 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f2732878f9b14c39899fd866a6341fd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033204 2024-11-13T19:43:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527095553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527095553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742023_1199 (size=14994) 2024-11-13T19:43:55,642 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#168 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:55,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/8ef962108ef349ae934ab786fe102a78 is 50, key is test_row_0/B:col10/1731527034175/Put/seqid=0 2024-11-13T19:43:55,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742025_1201 (size=31835) 2024-11-13T19:43:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742026_1202 (size=12881) 2024-11-13T19:43:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T19:43:55,734 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:55,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T19:43:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:55,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:55,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:55,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:55,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527095743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527095764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527095774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527095783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:55,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T19:43:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:55,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:56,005 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:56,030 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac1e91cfd0a94b4aaf55c8dacb172e87_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac1e91cfd0a94b4aaf55c8dacb172e87_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:56,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:56,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 is 175, key is test_row_0/A:col10/1731527034267/Put/seqid=0 2024-11-13T19:43:56,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742027_1203 (size=39949) 2024-11-13T19:43:56,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 2024-11-13T19:43:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:56,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T19:43:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:56,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527096054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/a3c3166b6a4d4355bb64459ba385c0a6 is 50, key is test_row_0/B:col10/1731527034267/Put/seqid=0 2024-11-13T19:43:56,074 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/2a665e137c1c464a8cea224f08f68e18 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18 2024-11-13T19:43:56,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527096083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,086 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into 2a665e137c1c464a8cea224f08f68e18(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:56,086 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:56,086 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=13, startTime=1731527035340; duration=0sec 2024-11-13T19:43:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527096083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,087 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:56,087 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:56,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742028_1204 (size=12301) 2024-11-13T19:43:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527096094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/a3c3166b6a4d4355bb64459ba385c0a6 2024-11-13T19:43:56,120 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/8ef962108ef349ae934ab786fe102a78 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/8ef962108ef349ae934ab786fe102a78 2024-11-13T19:43:56,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1ed9be7c78bc4cd9af34987bfef98138 is 50, key is test_row_0/C:col10/1731527034267/Put/seqid=0 2024-11-13T19:43:56,128 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 8ef962108ef349ae934ab786fe102a78(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:56,128 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:56,128 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=13, startTime=1731527035340; duration=0sec 2024-11-13T19:43:56,128 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:56,128 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742029_1205 (size=12301) 2024-11-13T19:43:56,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1ed9be7c78bc4cd9af34987bfef98138 2024-11-13T19:43:56,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 2024-11-13T19:43:56,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0, entries=200, sequenceid=296, filesize=39.0 K 2024-11-13T19:43:56,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/a3c3166b6a4d4355bb64459ba385c0a6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6 2024-11-13T19:43:56,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T19:43:56,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/1ed9be7c78bc4cd9af34987bfef98138 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138 2024-11-13T19:43:56,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T19:43:56,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for da41fb4884d60614ee8b9d1f2d3eb0fa in 804ms, sequenceid=296, compaction requested=false 2024-11-13T19:43:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:56,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T19:43:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:56,209 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:43:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:43:56,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135925be81b22945a9b0759fef9d0b065a_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527035395/Put/seqid=0 2024-11-13T19:43:56,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742030_1206 (size=12454) 2024-11-13T19:43:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:56,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527096592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527096592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527096593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527096593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527096604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527096707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527096707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,718 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135925be81b22945a9b0759fef9d0b065a_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135925be81b22945a9b0759fef9d0b065a_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:56,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/c525a4eb8b5044d5a0f544fd98b4b4d5, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/c525a4eb8b5044d5a0f544fd98b4b4d5 is 175, key is test_row_0/A:col10/1731527035395/Put/seqid=0 2024-11-13T19:43:56,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742031_1207 (size=31255) 2024-11-13T19:43:56,744 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/c525a4eb8b5044d5a0f544fd98b4b4d5 2024-11-13T19:43:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/59a920d5687c41b7ab262c8c025374e0 is 50, key is test_row_0/B:col10/1731527035395/Put/seqid=0 2024-11-13T19:43:56,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742032_1208 (size=12301) 2024-11-13T19:43:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:56,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527096923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527096923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,174 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/59a920d5687c41b7ab262c8c025374e0 2024-11-13T19:43:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b01e29654458449cbaec7a572b6d1d6d is 50, key is test_row_0/C:col10/1731527035395/Put/seqid=0 2024-11-13T19:43:57,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742033_1209 (size=12301) 2024-11-13T19:43:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b01e29654458449cbaec7a572b6d1d6d 2024-11-13T19:43:57,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/c525a4eb8b5044d5a0f544fd98b4b4d5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5 2024-11-13T19:43:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527097234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527097234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,244 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5, entries=150, sequenceid=311, filesize=30.5 K 2024-11-13T19:43:57,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/59a920d5687c41b7ab262c8c025374e0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0 2024-11-13T19:43:57,273 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T19:43:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b01e29654458449cbaec7a572b6d1d6d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d 2024-11-13T19:43:57,300 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T19:43:57,302 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1093ms, sequenceid=311, compaction requested=true 2024-11-13T19:43:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T19:43:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T19:43:57,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T19:43:57,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-11-13T19:43:57,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5720 sec 2024-11-13T19:43:57,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527036531, max timestamp=1731527037608Num uniques 859; 2024-11-13T19:43:57,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:57,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527036531, max timestamp=1731527037608Num uniques 859; 2024-11-13T19:43:57,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:57,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527036531, max timestamp=1731527037608Num uniques 859; 2024-11-13T19:43:57,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:43:57,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-13T19:43:57,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:57,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:57,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:57,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:57,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:57,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:57,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135093679764254e26bd04b27fa22a91b8_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:57,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742034_1210 (size=17534) 2024-11-13T19:43:57,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527037663, max timestamp=1731527037691Num uniques 859; 2024-11-13T19:43:57,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:57,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527037663, max timestamp=1731527037691Num uniques 859; 2024-11-13T19:43:57,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:57,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527037663, max timestamp=1731527037691Num uniques 859; 2024-11-13T19:43:57,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:43:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527097746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527097748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527097749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527097766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527097770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T19:43:57,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527097883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,884 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:43:57,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527097883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527097883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:57,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:43:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T19:43:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:43:57,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:43:57,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:43:57,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:43:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:43:58,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:58,067 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135093679764254e26bd04b27fa22a91b8_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135093679764254e26bd04b27fa22a91b8_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:58,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/d66de962408049f3a3335e1b717cb66e, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:58,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/d66de962408049f3a3335e1b717cb66e is 175, key is test_row_0/A:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527098093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527098093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527098094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742035_1211 (size=48639) 2024-11-13T19:43:58,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:43:58,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527098404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527098404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527098404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/d66de962408049f3a3335e1b717cb66e 2024-11-13T19:43:58,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:43:58,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/71e32fbd388a433690f3112bfd5bc558 is 50, key is test_row_0/B:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:58,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742036_1212 (size=12301) 2024-11-13T19:43:58,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/71e32fbd388a433690f3112bfd5bc558 2024-11-13T19:43:58,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/59dacd77f3bc4d1a804086d01b3d8dd7 is 50, key is test_row_0/C:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742037_1213 (size=12301) 2024-11-13T19:43:58,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527098774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,783 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527098789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,797 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in da41fb4884d60614ee8b9d1f2d3eb0fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:43:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527098914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527098914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527098915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:58,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:58,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:43:58,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/59dacd77f3bc4d1a804086d01b3d8dd7 2024-11-13T19:43:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/d66de962408049f3a3335e1b717cb66e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e 2024-11-13T19:43:59,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e, entries=250, sequenceid=336, filesize=47.5 K 2024-11-13T19:43:59,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/71e32fbd388a433690f3112bfd5bc558 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558 2024-11-13T19:43:59,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T19:43:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:43:59,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/59dacd77f3bc4d1a804086d01b3d8dd7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7 2024-11-13T19:43:59,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T19:43:59,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1414ms, sequenceid=336, compaction requested=true 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:59,035 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:43:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:43:59,037 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:59,039 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:59,039 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:43:59,039 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:59,039 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=148.1 K 2024-11-13T19:43:59,039 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:59,039 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e] 2024-11-13T19:43:59,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a665e137c1c464a8cea224f08f68e18, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033204 2024-11-13T19:43:59,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a9b9c5fe344ee5a6e9f67bf86ff1a0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527034226 2024-11-13T19:43:59,041 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c525a4eb8b5044d5a0f544fd98b4b4d5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731527035395 2024-11-13T19:43:59,049 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d66de962408049f3a3335e1b717cb66e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:43:59,050 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:59,050 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:43:59,050 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:59,050 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/8ef962108ef349ae934ab786fe102a78, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=48.6 K 2024-11-13T19:43:59,050 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef962108ef349ae934ab786fe102a78, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033204 2024-11-13T19:43:59,051 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c3166b6a4d4355bb64459ba385c0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527034267 2024-11-13T19:43:59,052 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a920d5687c41b7ab262c8c025374e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731527035395 2024-11-13T19:43:59,053 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e32fbd388a433690f3112bfd5bc558, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:43:59,062 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:59,065 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411137f543c7b7b7342d6a979612ccf7c082f_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:59,067 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411137f543c7b7b7342d6a979612ccf7c082f_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:59,067 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137f543c7b7b7342d6a979612ccf7c082f_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:59,080 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:59,080 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/65b94eb957c74aaf834737d6adaa3143 is 50, key is test_row_0/B:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:59,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742038_1214 (size=4469) 2024-11-13T19:43:59,094 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#177 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:59,095 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43d49698c9a9412ba30f4e63a6f075b4 is 175, key is test_row_0/A:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:59,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742039_1215 (size=13017) 2024-11-13T19:43:59,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742040_1216 (size=31971) 2024-11-13T19:43:59,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:43:59,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:59,137 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:43:59,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:43:59,150 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/43d49698c9a9412ba30f4e63a6f075b4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4 2024-11-13T19:43:59,163 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into 43d49698c9a9412ba30f4e63a6f075b4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:59,163 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:59,163 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=12, startTime=1731527039035; duration=0sec 2024-11-13T19:43:59,164 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:43:59,164 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:43:59,164 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:43:59,165 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:43:59,165 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:43:59,165 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:43:59,165 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/ddec1ec315664d71bb7788a6d9a61472, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=48.6 K 2024-11-13T19:43:59,166 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ddec1ec315664d71bb7788a6d9a61472, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731527033204 2024-11-13T19:43:59,166 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed9be7c78bc4cd9af34987bfef98138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527034267 2024-11-13T19:43:59,166 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b01e29654458449cbaec7a572b6d1d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731527035395 2024-11-13T19:43:59,166 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dacd77f3bc4d1a804086d01b3d8dd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:43:59,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135e7c8c6924254a69b816062ee224b340_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527037752/Put/seqid=0 2024-11-13T19:43:59,191 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#180 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:43:59,191 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/abf7943be1bf4e9d9cce31b2268739b7 is 50, key is test_row_0/C:col10/1731527037620/Put/seqid=0 2024-11-13T19:43:59,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742041_1217 (size=12454) 2024-11-13T19:43:59,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742042_1218 (size=13017) 2024-11-13T19:43:59,534 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/65b94eb957c74aaf834737d6adaa3143 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/65b94eb957c74aaf834737d6adaa3143 2024-11-13T19:43:59,540 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 65b94eb957c74aaf834737d6adaa3143(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:59,540 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:59,540 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=12, startTime=1731527039035; duration=0sec 2024-11-13T19:43:59,540 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:59,540 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:43:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:43:59,635 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135e7c8c6924254a69b816062ee224b340_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e7c8c6924254a69b816062ee224b340_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b4a0b94d62644db49f4d68e387bc3944, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:43:59,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b4a0b94d62644db49f4d68e387bc3944 is 175, key is test_row_0/A:col10/1731527037752/Put/seqid=0 2024-11-13T19:43:59,644 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/abf7943be1bf4e9d9cce31b2268739b7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/abf7943be1bf4e9d9cce31b2268739b7 2024-11-13T19:43:59,656 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into abf7943be1bf4e9d9cce31b2268739b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:43:59,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:43:59,656 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=12, startTime=1731527039035; duration=0sec 2024-11-13T19:43:59,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:43:59,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:43:59,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742043_1219 (size=31255) 2024-11-13T19:43:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. as already flushing 2024-11-13T19:43:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:43:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527099944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:59,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527099946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:59,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:43:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527099953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:43:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:43:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:44:00,063 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b4a0b94d62644db49f4d68e387bc3944 2024-11-13T19:44:00,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527100063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527100064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527100064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bed543b0f1cf48399fb8458ef541cdd7 is 50, key is test_row_0/B:col10/1731527037752/Put/seqid=0 2024-11-13T19:44:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742044_1220 (size=12301) 2024-11-13T19:44:00,090 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bed543b0f1cf48399fb8458ef541cdd7 2024-11-13T19:44:00,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/866ff1bb6fa94b14a2aaa2e62f7b184c is 50, key is test_row_0/C:col10/1731527037752/Put/seqid=0 2024-11-13T19:44:00,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742045_1221 (size=12301) 2024-11-13T19:44:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527100274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527100273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527100274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,517 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/866ff1bb6fa94b14a2aaa2e62f7b184c 2024-11-13T19:44:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/b4a0b94d62644db49f4d68e387bc3944 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944 2024-11-13T19:44:00,540 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944, entries=150, sequenceid=372, filesize=30.5 K 2024-11-13T19:44:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/bed543b0f1cf48399fb8458ef541cdd7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7 2024-11-13T19:44:00,547 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T19:44:00,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/866ff1bb6fa94b14a2aaa2e62f7b184c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c 2024-11-13T19:44:00,554 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T19:44:00,555 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1418ms, sequenceid=372, compaction requested=false 2024-11-13T19:44:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T19:44:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T19:44:00,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T19:44:00,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6680 sec 2024-11-13T19:44:00,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6860 sec 2024-11-13T19:44:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:00,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:44:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:00,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113608cf0b50fb14939938e0405faf056d2_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742046_1222 (size=14994) 2024-11-13T19:44:00,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:00,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527040616, max timestamp=1731527040635Num uniques 859; 2024-11-13T19:44:00,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:44:00,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731527040616, max timestamp=1731527040635Num uniques 859; 2024-11-13T19:44:00,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527040616, max timestamp=1731527040635Num uniques 859; 2024-11-13T19:44:00,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:44:00,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:00,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113608cf0b50fb14939938e0405faf056d2_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113608cf0b50fb14939938e0405faf056d2_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:00,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6a88bfc48e8f44abb52fdcb4760864c3, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6a88bfc48e8f44abb52fdcb4760864c3 is 175, key is test_row_0/A:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742047_1223 (size=39949) 2024-11-13T19:44:00,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6a88bfc48e8f44abb52fdcb4760864c3 2024-11-13T19:44:00,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/d80f95d80aca4bacbb0f268c4606ea69 is 50, key is test_row_0/B:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527100683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742048_1224 (size=12301) 2024-11-13T19:44:00,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/d80f95d80aca4bacbb0f268c4606ea69 2024-11-13T19:44:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527100685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527100685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f266a8334bdc49249fc5a697447bbbbb is 50, key is test_row_0/C:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742049_1225 (size=12301) 2024-11-13T19:44:00,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f266a8334bdc49249fc5a697447bbbbb 2024-11-13T19:44:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/6a88bfc48e8f44abb52fdcb4760864c3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3 2024-11-13T19:44:00,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3, entries=200, sequenceid=385, filesize=39.0 K 2024-11-13T19:44:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/d80f95d80aca4bacbb0f268c4606ea69 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69 2024-11-13T19:44:00,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69, entries=150, sequenceid=385, filesize=12.0 K 2024-11-13T19:44:00,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/f266a8334bdc49249fc5a697447bbbbb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb 2024-11-13T19:44:00,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb, entries=150, sequenceid=385, filesize=12.0 K 2024-11-13T19:44:00,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for da41fb4884d60614ee8b9d1f2d3eb0fa in 202ms, sequenceid=385, compaction requested=true 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:00,789 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:00,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:00,790 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:00,790 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/A is initiating minor compaction (all files) 2024-11-13T19:44:00,790 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/A in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:00,790 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=100.8 K 2024-11-13T19:44:00,790 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:00,790 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3] 2024-11-13T19:44:00,791 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:00,791 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/B is initiating minor compaction (all files) 2024-11-13T19:44:00,791 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/B in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:00,791 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/65b94eb957c74aaf834737d6adaa3143, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.7 K 2024-11-13T19:44:00,791 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d49698c9a9412ba30f4e63a6f075b4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:44:00,791 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a0b94d62644db49f4d68e387bc3944, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731527037741 2024-11-13T19:44:00,791 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b94eb957c74aaf834737d6adaa3143, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:44:00,792 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a88bfc48e8f44abb52fdcb4760864c3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731527039937 2024-11-13T19:44:00,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bed543b0f1cf48399fb8458ef541cdd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731527037741 2024-11-13T19:44:00,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d80f95d80aca4bacbb0f268c4606ea69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731527039937 2024-11-13T19:44:00,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-13T19:44:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:00,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:44:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:44:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:44:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:00,806 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527100808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527100810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527100810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,816 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:00,816 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/48858fa3777f4a55b3f4c84854fa9ffd is 50, key is test_row_0/B:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527100812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,817 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d043cc0d07d944fd9c3db5103e970dbc_da41fb4884d60614ee8b9d1f2d3eb0fa store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527100812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,819 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d043cc0d07d944fd9c3db5103e970dbc_da41fb4884d60614ee8b9d1f2d3eb0fa, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,820 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d043cc0d07d944fd9c3db5103e970dbc_da41fb4884d60614ee8b9d1f2d3eb0fa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111311aee3631d0548b1b4d254948025ccb0_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527040671/Put/seqid=0 2024-11-13T19:44:00,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742050_1226 (size=13119) 2024-11-13T19:44:00,841 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/48858fa3777f4a55b3f4c84854fa9ffd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/48858fa3777f4a55b3f4c84854fa9ffd 2024-11-13T19:44:00,857 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/B of da41fb4884d60614ee8b9d1f2d3eb0fa into 48858fa3777f4a55b3f4c84854fa9ffd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:00,857 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:00,857 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/B, priority=13, startTime=1731527040789; duration=0sec 2024-11-13T19:44:00,857 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:00,857 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:44:00,858 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:00,860 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:00,860 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): da41fb4884d60614ee8b9d1f2d3eb0fa/C is initiating minor compaction (all files) 2024-11-13T19:44:00,860 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da41fb4884d60614ee8b9d1f2d3eb0fa/C in TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:00,860 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/abf7943be1bf4e9d9cce31b2268739b7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp, totalSize=36.7 K 2024-11-13T19:44:00,860 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting abf7943be1bf4e9d9cce31b2268739b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731527036586 2024-11-13T19:44:00,861 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 866ff1bb6fa94b14a2aaa2e62f7b184c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731527037741 2024-11-13T19:44:00,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742051_1227 (size=4469) 2024-11-13T19:44:00,862 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f266a8334bdc49249fc5a697447bbbbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731527039937 2024-11-13T19:44:00,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742052_1228 (size=12454) 2024-11-13T19:44:00,878 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#C#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:00,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:00,879 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/499033f91f264f529b5f1983b73e66a4 is 50, key is test_row_0/C:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:00,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111311aee3631d0548b1b4d254948025ccb0_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111311aee3631d0548b1b4d254948025ccb0_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:00,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/cb54a259a764447d8912b8f7cd4d6832, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:00,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/cb54a259a764447d8912b8f7cd4d6832 is 175, key is test_row_0/A:col10/1731527040671/Put/seqid=0 2024-11-13T19:44:00,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742053_1229 (size=13119) 2024-11-13T19:44:00,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742054_1230 (size=31255) 2024-11-13T19:44:00,920 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/499033f91f264f529b5f1983b73e66a4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/499033f91f264f529b5f1983b73e66a4 2024-11-13T19:44:00,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527100924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527100925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527100926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527100926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527100926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:00,933 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/C of da41fb4884d60614ee8b9d1f2d3eb0fa into 499033f91f264f529b5f1983b73e66a4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:00,933 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:00,933 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/C, priority=13, startTime=1731527040789; duration=0sec 2024-11-13T19:44:00,933 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:00,933 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:44:01,017 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-13T19:44:01,017 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:01,017 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,017 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:01,018 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-13T19:44:01,018 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:01,018 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,018 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:01,024 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-13T19:44:01,024 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:01,024 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,024 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:01,032 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-13T19:44:01,032 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:01,032 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,032 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:01,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527101134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527101134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527101134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527101134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527101134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,261 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da41fb4884d60614ee8b9d1f2d3eb0fa#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:01,261 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/fd8e508ac10a46a8a8cb7cb39b903bd3 is 175, key is test_row_0/A:col10/1731527040585/Put/seqid=0 2024-11-13T19:44:01,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742055_1231 (size=32073) 2024-11-13T19:44:01,313 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/cb54a259a764447d8912b8f7cd4d6832 2024-11-13T19:44:01,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/198fcbe5f2f34ed98c7624883b9a4485 is 50, key is test_row_0/B:col10/1731527040671/Put/seqid=0 2024-11-13T19:44:01,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742056_1232 (size=12301) 2024-11-13T19:44:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527101444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527101444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527101444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527101444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527101444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,673 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/fd8e508ac10a46a8a8cb7cb39b903bd3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fd8e508ac10a46a8a8cb7cb39b903bd3 2024-11-13T19:44:01,679 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da41fb4884d60614ee8b9d1f2d3eb0fa/A of da41fb4884d60614ee8b9d1f2d3eb0fa into fd8e508ac10a46a8a8cb7cb39b903bd3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:01,679 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:01,679 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., storeName=da41fb4884d60614ee8b9d1f2d3eb0fa/A, priority=13, startTime=1731527040789; duration=0sec 2024-11-13T19:44:01,679 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:01,679 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:44:01,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/198fcbe5f2f34ed98c7624883b9a4485 2024-11-13T19:44:01,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/472110631e28413ca9489ea00d5e2c8d is 50, key is test_row_0/C:col10/1731527040671/Put/seqid=0 2024-11-13T19:44:01,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742057_1233 (size=12301) 2024-11-13T19:44:01,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60016 deadline: 1731527101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60056 deadline: 1731527101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60044 deadline: 1731527101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731527101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60004 deadline: 1731527101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:44:02,054 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:02,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/472110631e28413ca9489ea00d5e2c8d 2024-11-13T19:44:02,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/cb54a259a764447d8912b8f7cd4d6832 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/cb54a259a764447d8912b8f7cd4d6832 2024-11-13T19:44:02,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/cb54a259a764447d8912b8f7cd4d6832, entries=150, sequenceid=422, filesize=30.5 K 2024-11-13T19:44:02,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/198fcbe5f2f34ed98c7624883b9a4485 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/198fcbe5f2f34ed98c7624883b9a4485 2024-11-13T19:44:02,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/198fcbe5f2f34ed98c7624883b9a4485, entries=150, sequenceid=422, filesize=12.0 K 2024-11-13T19:44:02,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/472110631e28413ca9489ea00d5e2c8d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/472110631e28413ca9489ea00d5e2c8d 2024-11-13T19:44:02,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/472110631e28413ca9489ea00d5e2c8d, entries=150, sequenceid=422, filesize=12.0 K 2024-11-13T19:44:02,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1369ms, sequenceid=422, compaction requested=false 2024-11-13T19:44:02,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:02,967 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-13T19:44:02,967 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-13T19:44:02,967 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:02,967 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:02,967 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-13T19:44:02,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,967 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:02,967 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,967 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:44:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:02,976 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-13T19:44:02,976 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:02,976 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,976 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,977 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-13T19:44:02,977 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:02,977 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,977 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2010 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1910 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 888 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2663 rows 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 882 2024-11-13T19:44:02,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2644 rows 2024-11-13T19:44:02,978 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:44:02,978 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:44:02,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:02,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:02,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:44:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:02,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113629f628bcba54db78b642b6026d03b3a_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527042964/Put/seqid=0 2024-11-13T19:44:02,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527042984"}]},"ts":"1731527042984"} 2024-11-13T19:44:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:02,987 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:44:02,987 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:44:02,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:44:02,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742058_1234 (size=12454) 2024-11-13T19:44:02,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, UNASSIGN}] 2024-11-13T19:44:02,999 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, UNASSIGN 2024-11-13T19:44:02,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:03,000 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:03,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, UNASSIGN because future has completed 2024-11-13T19:44:03,005 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:44:03,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:03,021 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113629f628bcba54db78b642b6026d03b3a_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113629f628bcba54db78b642b6026d03b3a_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:03,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/e06bab0042f5423a8b1145d81e491e18, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:03,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/e06bab0042f5423a8b1145d81e491e18 is 175, key is test_row_0/A:col10/1731527042964/Put/seqid=0 2024-11-13T19:44:03,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742059_1235 (size=31255) 2024-11-13T19:44:03,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/e06bab0042f5423a8b1145d81e491e18 2024-11-13T19:44:03,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c575392dd388432bb491d98316d00de0 is 50, key is test_row_0/B:col10/1731527042964/Put/seqid=0 2024-11-13T19:44:03,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742060_1236 (size=12301) 2024-11-13T19:44:03,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c575392dd388432bb491d98316d00de0 2024-11-13T19:44:03,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/9bdfd4b8599145188eea14f22e7de474 is 50, key is test_row_0/C:col10/1731527042964/Put/seqid=0 2024-11-13T19:44:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:03,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742061_1237 (size=12301) 2024-11-13T19:44:03,162 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(122): Close da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:03,162 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:44:03,162 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1722): Closing da41fb4884d60614ee8b9d1f2d3eb0fa, disabling compactions & flushes 2024-11-13T19:44:03,162 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:03,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/9bdfd4b8599145188eea14f22e7de474 2024-11-13T19:44:03,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/e06bab0042f5423a8b1145d81e491e18 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/e06bab0042f5423a8b1145d81e491e18 2024-11-13T19:44:03,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/e06bab0042f5423a8b1145d81e491e18, entries=150, sequenceid=437, filesize=30.5 K 2024-11-13T19:44:03,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/c575392dd388432bb491d98316d00de0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c575392dd388432bb491d98316d00de0 2024-11-13T19:44:03,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c575392dd388432bb491d98316d00de0, entries=150, sequenceid=437, filesize=12.0 K 2024-11-13T19:44:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/9bdfd4b8599145188eea14f22e7de474 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/9bdfd4b8599145188eea14f22e7de474 2024-11-13T19:44:03,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/9bdfd4b8599145188eea14f22e7de474, entries=150, sequenceid=437, filesize=12.0 K 2024-11-13T19:44:03,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for da41fb4884d60614ee8b9d1f2d3eb0fa in 549ms, sequenceid=437, compaction requested=true 2024-11-13T19:44:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da41fb4884d60614ee8b9d1f2d3eb0fa: 2024-11-13T19:44:03,518 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. after waiting 0 ms 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. because compaction request was cancelled 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:A 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da41fb4884d60614ee8b9d1f2d3eb0fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. because compaction request was cancelled 2024-11-13T19:44:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. because compaction request was cancelled 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:C 2024-11-13T19:44:03,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da41fb4884d60614ee8b9d1f2d3eb0fa:B 2024-11-13T19:44:03,519 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2902): Flushing da41fb4884d60614ee8b9d1f2d3eb0fa 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=A 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=B 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da41fb4884d60614ee8b9d1f2d3eb0fa, store=C 2024-11-13T19:44:03,519 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:03,526 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111319db0a88cff640889d7be284a56c4a83_da41fb4884d60614ee8b9d1f2d3eb0fa is 50, key is test_row_0/A:col10/1731527042975/Put/seqid=0 2024-11-13T19:44:03,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742062_1238 (size=9914) 2024-11-13T19:44:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:03,932 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:03,937 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111319db0a88cff640889d7be284a56c4a83_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111319db0a88cff640889d7be284a56c4a83_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:03,938 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/7ed166aae9324866b15db16ac2efec19, store: [table=TestAcidGuarantees family=A region=da41fb4884d60614ee8b9d1f2d3eb0fa] 2024-11-13T19:44:03,939 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/7ed166aae9324866b15db16ac2efec19 is 175, key is test_row_0/A:col10/1731527042975/Put/seqid=0 2024-11-13T19:44:03,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742063_1239 (size=22561) 2024-11-13T19:44:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:04,344 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/7ed166aae9324866b15db16ac2efec19 2024-11-13T19:44:04,354 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f85aa5f45d0f4a24a5eaf4188091e9da is 50, key is test_row_0/B:col10/1731527042975/Put/seqid=0 2024-11-13T19:44:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742064_1240 (size=9857) 2024-11-13T19:44:04,761 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f85aa5f45d0f4a24a5eaf4188091e9da 2024-11-13T19:44:04,771 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b7d1e9d9ca3f4710806d39b487ac9cfd is 50, key is test_row_0/C:col10/1731527042975/Put/seqid=0 2024-11-13T19:44:04,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742065_1241 (size=9857) 2024-11-13T19:44:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:05,181 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b7d1e9d9ca3f4710806d39b487ac9cfd 2024-11-13T19:44:05,190 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/A/7ed166aae9324866b15db16ac2efec19 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/7ed166aae9324866b15db16ac2efec19 2024-11-13T19:44:05,195 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/7ed166aae9324866b15db16ac2efec19, entries=100, sequenceid=442, filesize=22.0 K 2024-11-13T19:44:05,197 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/B/f85aa5f45d0f4a24a5eaf4188091e9da as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f85aa5f45d0f4a24a5eaf4188091e9da 2024-11-13T19:44:05,210 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f85aa5f45d0f4a24a5eaf4188091e9da, entries=100, sequenceid=442, filesize=9.6 K 2024-11-13T19:44:05,212 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/.tmp/C/b7d1e9d9ca3f4710806d39b487ac9cfd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b7d1e9d9ca3f4710806d39b487ac9cfd 2024-11-13T19:44:05,218 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b7d1e9d9ca3f4710806d39b487ac9cfd, entries=100, sequenceid=442, filesize=9.6 K 2024-11-13T19:44:05,219 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1700ms, sequenceid=442, compaction requested=true 2024-11-13T19:44:05,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3] to archive 2024-11-13T19:44:05,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:05,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/1f5e19159a29401885fd50183d83de00 2024-11-13T19:44:05,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/08ade9a1508e4490bf037f75640c657a 2024-11-13T19:44:05,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/16139d9a877a41fc8885af94b751254f 2024-11-13T19:44:05,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/f4a5ad5b62664a638577c681b59f1f71 2024-11-13T19:44:05,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/151ee9fe6dd74f88a43ab03c4f2489b6 2024-11-13T19:44:05,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/025e272dd48c4202b0800f81ec7201f2 2024-11-13T19:44:05,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/dca623b62f6d48768a7fb6ded61669ab 2024-11-13T19:44:05,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/54f3cf16ec2b4e60b41f2aa88174cbb0 2024-11-13T19:44:05,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/17527cb5d50f4272bdd61460a1e91c27 2024-11-13T19:44:05,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a7967be256974044a6c81fc5553be3aa 2024-11-13T19:44:05,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/bb0022ddd9c44db79d84a6165f689f57 2024-11-13T19:44:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b535dbf27bfa4fd09e5ba278248b6bb5 2024-11-13T19:44:05,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43e1e85336424e9096794267b074d77f 2024-11-13T19:44:05,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6ffd3dfd08254e4385ee88d50adfb017 2024-11-13T19:44:05,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fb78ed248779448ba3505d071cc440d3 2024-11-13T19:44:05,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3553aada894b5faeaeb25c259c8ff5 2024-11-13T19:44:05,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/ee3312351bdf48efa01cc5a8a7a80d99 2024-11-13T19:44:05,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/2a665e137c1c464a8cea224f08f68e18 2024-11-13T19:44:05,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/a8a9b9c5fe344ee5a6e9f67bf86ff1a0 2024-11-13T19:44:05,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/c525a4eb8b5044d5a0f544fd98b4b4d5 2024-11-13T19:44:05,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/d66de962408049f3a3335e1b717cb66e 2024-11-13T19:44:05,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/43d49698c9a9412ba30f4e63a6f075b4 2024-11-13T19:44:05,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/b4a0b94d62644db49f4d68e387bc3944 2024-11-13T19:44:05,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/6a88bfc48e8f44abb52fdcb4760864c3 2024-11-13T19:44:05,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6e3404fba3cd490297b87d1e7188b78b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/84ec4ff4eefd4c329077bdf459cda778, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6df7fe1bd4b84a92af1d0fa365b24330, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/96e3d62eea024c8db7f6fb372a61e66a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/8ef962108ef349ae934ab786fe102a78, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/65b94eb957c74aaf834737d6adaa3143, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69] to archive 2024-11-13T19:44:05,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:05,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/82654f1b0bb8430ba2ddb99ffab380ae 2024-11-13T19:44:05,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e538a0bdfad0433c8a7ed7a803bdabd8 2024-11-13T19:44:05,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/9185adeafe6e4de0a466541eb5356a53 2024-11-13T19:44:05,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6e3404fba3cd490297b87d1e7188b78b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6e3404fba3cd490297b87d1e7188b78b 2024-11-13T19:44:05,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c63791c10575474fa52824b2878c0a15 2024-11-13T19:44:05,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c7926540bcca48ab9dd5a0317765bd87 2024-11-13T19:44:05,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/93de25d99cc2408fbee408ace544e8a8 2024-11-13T19:44:05,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/84ec4ff4eefd4c329077bdf459cda778 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/84ec4ff4eefd4c329077bdf459cda778 2024-11-13T19:44:05,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bc86baef6a614542927483e05105334e 2024-11-13T19:44:05,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/647d8b4886c741f2a270cde580104dd4 2024-11-13T19:44:05,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6df7fe1bd4b84a92af1d0fa365b24330 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/6df7fe1bd4b84a92af1d0fa365b24330 2024-11-13T19:44:05,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c1dabb92fc1d4a35aa50b31ff8f4c37f 2024-11-13T19:44:05,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/e7287dde078445b2b3ccb1b02f7b7032 2024-11-13T19:44:05,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/96e3d62eea024c8db7f6fb372a61e66a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/96e3d62eea024c8db7f6fb372a61e66a 2024-11-13T19:44:05,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/4c96f64dff7b4d4b8876774af8622c8a 2024-11-13T19:44:05,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/2bc6b8c763ee4f40a966038e7507691a 2024-11-13T19:44:05,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/8ef962108ef349ae934ab786fe102a78 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/8ef962108ef349ae934ab786fe102a78 2024-11-13T19:44:05,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f2732878f9b14c39899fd866a6341fd7 2024-11-13T19:44:05,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/a3c3166b6a4d4355bb64459ba385c0a6 2024-11-13T19:44:05,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/59a920d5687c41b7ab262c8c025374e0 2024-11-13T19:44:05,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/65b94eb957c74aaf834737d6adaa3143 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/65b94eb957c74aaf834737d6adaa3143 2024-11-13T19:44:05,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/71e32fbd388a433690f3112bfd5bc558 2024-11-13T19:44:05,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/bed543b0f1cf48399fb8458ef541cdd7 2024-11-13T19:44:05,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/d80f95d80aca4bacbb0f268c4606ea69 2024-11-13T19:44:05,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/8d8644e0355a4696b8b57e5d3149782f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/a609750091ce421ca3919b30493c5be1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/7d30459e3f8e4e789ab10c53dc642701, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/68140f81cc434202878e03e2a02eeeac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/ddec1ec315664d71bb7788a6d9a61472, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/abf7943be1bf4e9d9cce31b2268739b7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb] to archive 2024-11-13T19:44:05,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:05,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dd8cb47daa5646699204b8e90dcdfe2b 2024-11-13T19:44:05,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/85aa56f996324e0e818d377914cd4fd8 2024-11-13T19:44:05,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1cf74f61e1404f38aaf39e454c72098f 2024-11-13T19:44:05,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/8d8644e0355a4696b8b57e5d3149782f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/8d8644e0355a4696b8b57e5d3149782f 2024-11-13T19:44:05,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/71f7144dcd034fbb87df7495d9889c76 2024-11-13T19:44:05,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/643c0cca20f04437978063f997996c75 2024-11-13T19:44:05,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/dce3b0663d26495e8e69c495e31be1ee 2024-11-13T19:44:05,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/a609750091ce421ca3919b30493c5be1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/a609750091ce421ca3919b30493c5be1 2024-11-13T19:44:05,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/6b953ae445c84ff79a38d3328df80eb5 2024-11-13T19:44:05,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/432d01efe50c4b9889616ceaea2b8405 2024-11-13T19:44:05,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/7d30459e3f8e4e789ab10c53dc642701 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/7d30459e3f8e4e789ab10c53dc642701 2024-11-13T19:44:05,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b5cf509e6e884c16ad8861b85dd63f45 2024-11-13T19:44:05,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d8ae9736f4dd42949f3b7e70d1fa1965 2024-11-13T19:44:05,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/68140f81cc434202878e03e2a02eeeac to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/68140f81cc434202878e03e2a02eeeac 2024-11-13T19:44:05,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/0c03b7bb99584ed8b2357f4a9b00fe59 2024-11-13T19:44:05,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f07ead95d74143baa766a89d24847e7c 2024-11-13T19:44:05,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/ddec1ec315664d71bb7788a6d9a61472 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/ddec1ec315664d71bb7788a6d9a61472 2024-11-13T19:44:05,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/d9af16aa56b84ea293e68ccda39cec6d 2024-11-13T19:44:05,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/1ed9be7c78bc4cd9af34987bfef98138 2024-11-13T19:44:05,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b01e29654458449cbaec7a572b6d1d6d 2024-11-13T19:44:05,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/abf7943be1bf4e9d9cce31b2268739b7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/abf7943be1bf4e9d9cce31b2268739b7 2024-11-13T19:44:05,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/59dacd77f3bc4d1a804086d01b3d8dd7 2024-11-13T19:44:05,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/866ff1bb6fa94b14a2aaa2e62f7b184c 2024-11-13T19:44:05,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/f266a8334bdc49249fc5a697447bbbbb 2024-11-13T19:44:05,446 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=4 2024-11-13T19:44:05,446 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. 2024-11-13T19:44:05,447 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1676): Region close journal for da41fb4884d60614ee8b9d1f2d3eb0fa: Waiting for close lock at 1731527043162Running coprocessor pre-close hooks at 1731527043162Disabling compacts and flushes for region at 1731527043162Disabling writes for close at 1731527043519 (+357 ms)Obtaining lock to block concurrent updates at 1731527043519Preparing flush snapshotting stores in da41fb4884d60614ee8b9d1f2d3eb0fa at 1731527043519Finished memstore snapshotting TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731527043519Flushing stores of TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa. at 1731527043520 (+1 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/A: creating writer at 1731527043520Flushing mob file da41fb4884d60614ee8b9d1f2d3eb0fa/A: appending metadata at 1731527043526 (+6 ms)Flushing mob file da41fb4884d60614ee8b9d1f2d3eb0fa/A: closing flushed file at 1731527043526Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/A: appending metadata at 1731527043938 (+412 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/A: closing flushed file at 1731527043938Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/B: creating writer at 1731527044350 (+412 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/B: appending metadata at 1731527044353 (+3 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/B: closing flushed file at 1731527044353Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/C: creating writer at 1731527044767 (+414 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/C: appending metadata at 1731527044770 (+3 ms)Flushing da41fb4884d60614ee8b9d1f2d3eb0fa/C: closing flushed file at 1731527044771 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7669189a: reopening flushed file at 1731527045189 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@302af0a6: reopening flushed file at 1731527045195 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7379aa38: reopening flushed file at 1731527045211 (+16 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for da41fb4884d60614ee8b9d1f2d3eb0fa in 1700ms, sequenceid=442, compaction requested=true at 1731527045219 (+8 ms)Writing region close event to WAL at 1731527045442 (+223 ms)Running coprocessor post-close hooks at 1731527045446 (+4 ms)Closed at 1731527045446 2024-11-13T19:44:05,450 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(157): Closed da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:05,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=da41fb4884d60614ee8b9d1f2d3eb0fa, regionState=CLOSED 2024-11-13T19:44:05,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:05,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T19:44:05,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseRegionProcedure da41fb4884d60614ee8b9d1f2d3eb0fa, server=54d0784d1943,36227,1731526989835 in 2.4570 sec 2024-11-13T19:44:05,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-11-13T19:44:05,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da41fb4884d60614ee8b9d1f2d3eb0fa, UNASSIGN in 2.4680 sec 2024-11-13T19:44:05,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T19:44:05,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4750 sec 2024-11-13T19:44:05,472 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527045471"}]},"ts":"1731527045471"} 2024-11-13T19:44:05,474 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:44:05,474 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:44:05,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4940 sec 2024-11-13T19:44:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:44:07,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:07,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:44:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,157 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:44:07,158 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=48, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,161 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits] 2024-11-13T19:44:07,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/7ed166aae9324866b15db16ac2efec19 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/7ed166aae9324866b15db16ac2efec19 2024-11-13T19:44:07,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/cb54a259a764447d8912b8f7cd4d6832 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/cb54a259a764447d8912b8f7cd4d6832 2024-11-13T19:44:07,171 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/e06bab0042f5423a8b1145d81e491e18 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/e06bab0042f5423a8b1145d81e491e18 2024-11-13T19:44:07,172 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fd8e508ac10a46a8a8cb7cb39b903bd3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/A/fd8e508ac10a46a8a8cb7cb39b903bd3 2024-11-13T19:44:07,176 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/198fcbe5f2f34ed98c7624883b9a4485 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/198fcbe5f2f34ed98c7624883b9a4485 2024-11-13T19:44:07,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/48858fa3777f4a55b3f4c84854fa9ffd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/48858fa3777f4a55b3f4c84854fa9ffd 2024-11-13T19:44:07,180 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c575392dd388432bb491d98316d00de0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/c575392dd388432bb491d98316d00de0 2024-11-13T19:44:07,181 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f85aa5f45d0f4a24a5eaf4188091e9da to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/B/f85aa5f45d0f4a24a5eaf4188091e9da 2024-11-13T19:44:07,184 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/472110631e28413ca9489ea00d5e2c8d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/472110631e28413ca9489ea00d5e2c8d 2024-11-13T19:44:07,185 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/499033f91f264f529b5f1983b73e66a4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/499033f91f264f529b5f1983b73e66a4 2024-11-13T19:44:07,187 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/9bdfd4b8599145188eea14f22e7de474 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/9bdfd4b8599145188eea14f22e7de474 2024-11-13T19:44:07,188 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b7d1e9d9ca3f4710806d39b487ac9cfd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/C/b7d1e9d9ca3f4710806d39b487ac9cfd 2024-11-13T19:44:07,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits/445.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa/recovered.edits/445.seqid 2024-11-13T19:44:07,194 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,194 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:44:07,194 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:44:07,195 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:44:07,202 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130814f09867e34fd5a9cf2ecd0994eba2_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130814f09867e34fd5a9cf2ecd0994eba2_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,204 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111311aee3631d0548b1b4d254948025ccb0_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111311aee3631d0548b1b4d254948025ccb0_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,206 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111319db0a88cff640889d7be284a56c4a83_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111319db0a88cff640889d7be284a56c4a83_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,208 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339b9e748d82b43c08addf9dc5d8cff97_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339b9e748d82b43c08addf9dc5d8cff97_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,209 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ad58def32b74a32a0cfda223d1a7842_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ad58def32b74a32a0cfda223d1a7842_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,211 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135093679764254e26bd04b27fa22a91b8_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135093679764254e26bd04b27fa22a91b8_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,212 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135925be81b22945a9b0759fef9d0b065a_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135925be81b22945a9b0759fef9d0b065a_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,214 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e7c8c6924254a69b816062ee224b340_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e7c8c6924254a69b816062ee224b340_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,215 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113608cf0b50fb14939938e0405faf056d2_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113608cf0b50fb14939938e0405faf056d2_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,216 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113629f628bcba54db78b642b6026d03b3a_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113629f628bcba54db78b642b6026d03b3a_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,218 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113639bd22130504461a336fca7570f2cbe_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113639bd22130504461a336fca7570f2cbe_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,219 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c1998a8247c41b2a4e31f92392a1b25_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c1998a8247c41b2a4e31f92392a1b25_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,221 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aecb96f0237430cb65f89eb1c07ac1e_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139aecb96f0237430cb65f89eb1c07ac1e_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,222 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac1e91cfd0a94b4aaf55c8dacb172e87_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac1e91cfd0a94b4aaf55c8dacb172e87_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,223 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1885e5f848d4db7a979d02035b0f0b7_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1885e5f848d4db7a979d02035b0f0b7_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,224 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c16d33504bd545e68c3b797aa39d2134_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c16d33504bd545e68c3b797aa39d2134_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c43429c6b4fe445dbff5d9acf142049e_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c43429c6b4fe445dbff5d9acf142049e_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,227 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d13718b129634203a36dbb0840b82fa1_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d13718b129634203a36dbb0840b82fa1_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d79af1e36ad346b38ac2af79d4aa7e08_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d79af1e36ad346b38ac2af79d4aa7e08_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dee51ae40906456bb59b8c37a2b8d280_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dee51ae40906456bb59b8c37a2b8d280_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e772c96c936a42b6823dc6cded941ac6_da41fb4884d60614ee8b9d1f2d3eb0fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e772c96c936a42b6823dc6cded941ac6_da41fb4884d60614ee8b9d1f2d3eb0fa 2024-11-13T19:44:07,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:44:07,235 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=48, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,238 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:44:07,241 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:44:07,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=48, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:44:07,243 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527047242"}]},"ts":"9223372036854775807"} 2024-11-13T19:44:07,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:44:07,245 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => da41fb4884d60614ee8b9d1f2d3eb0fa, NAME => 'TestAcidGuarantees,,1731527018859.da41fb4884d60614ee8b9d1f2d3eb0fa.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:44:07,245 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:44:07,245 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527047245"}]},"ts":"9223372036854775807"} 2024-11-13T19:44:07,247 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:44:07,248 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=48, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-13T19:44:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:44:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:44:07,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:07,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=254 (was 248) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_543150073_22 at /127.0.0.1:50608 [Waiting for operation #356] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1517952646_22 at /127.0.0.1:34420 [Waiting for operation #683] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_543150073_22 at /127.0.0.1:34502 [Waiting for operation #681] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1517952646_22 at /127.0.0.1:50604 [Waiting for operation #361] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=944 (was 897) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=523 (was 325) - AvailableMemoryMB LEAK? - 2024-11-13T19:44:07,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=254, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=944, ProcessCount=11, AvailableMemoryMB=523 2024-11-13T19:44:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:44:07,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:44:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:07,290 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:44:07,291 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:07,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 49 2024-11-13T19:44:07,291 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:44:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:07,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742066_1242 (size=1061) 2024-11-13T19:44:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:07,700 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:44:07,706 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:44:07,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742067_1243 (size=53) 2024-11-13T19:44:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6f4ececb48a6d1da14a168d390e345dd, disabling compactions & flushes 2024-11-13T19:44:08,109 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. after waiting 0 ms 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,109 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,109 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f4ececb48a6d1da14a168d390e345dd: Waiting for close lock at 1731527048109Disabling compacts and flushes for region at 1731527048109Disabling writes for close at 1731527048109Writing region close event to WAL at 1731527048109Closed at 1731527048109 2024-11-13T19:44:08,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:44:08,110 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731527048110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527048110"}]},"ts":"1731527048110"} 2024-11-13T19:44:08,112 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:44:08,113 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:44:08,113 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527048113"}]},"ts":"1731527048113"} 2024-11-13T19:44:08,114 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:44:08,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, ASSIGN}] 2024-11-13T19:44:08,115 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, ASSIGN 2024-11-13T19:44:08,116 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:44:08,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=6f4ececb48a6d1da14a168d390e345dd, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:08,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, ASSIGN because future has completed 2024-11-13T19:44:08,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:08,425 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,425 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:44:08,426 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,426 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:44:08,426 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,426 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,427 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,429 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:08,429 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f4ececb48a6d1da14a168d390e345dd columnFamilyName A 2024-11-13T19:44:08,429 DEBUG [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:08,430 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(327): Store=6f4ececb48a6d1da14a168d390e345dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:08,430 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,431 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:08,431 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f4ececb48a6d1da14a168d390e345dd columnFamilyName B 2024-11-13T19:44:08,431 DEBUG [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:08,431 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(327): Store=6f4ececb48a6d1da14a168d390e345dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:08,431 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,432 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:08,432 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f4ececb48a6d1da14a168d390e345dd columnFamilyName C 2024-11-13T19:44:08,432 DEBUG [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:08,433 INFO [StoreOpener-6f4ececb48a6d1da14a168d390e345dd-1 {}] regionserver.HStore(327): Store=6f4ececb48a6d1da14a168d390e345dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:08,433 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,433 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,434 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,434 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,435 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,435 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,435 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:44:08,436 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,438 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:44:08,439 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 6f4ececb48a6d1da14a168d390e345dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60554433, jitterRate=-0.09766863286495209}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:44:08,439 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:08,439 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 6f4ececb48a6d1da14a168d390e345dd: Running coprocessor pre-open hook at 1731527048426Writing region info on filesystem at 1731527048426Initializing all the Stores at 1731527048427 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527048427Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527048427Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527048427Cleaning up temporary data from old regions at 1731527048435 (+8 ms)Running coprocessor post-open hooks at 1731527048439 (+4 ms)Region opened successfully at 1731527048439 2024-11-13T19:44:08,440 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., pid=51, masterSystemTime=1731527048421 2024-11-13T19:44:08,442 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,442 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:08,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=6f4ececb48a6d1da14a168d390e345dd, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:08,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:08,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T19:44:08,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 in 176 msec 2024-11-13T19:44:08,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-13T19:44:08,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, ASSIGN in 333 msec 2024-11-13T19:44:08,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:44:08,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527048449"}]},"ts":"1731527048449"} 2024-11-13T19:44:08,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:44:08,452 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:44:08,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-13T19:44:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-13T19:44:09,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:09,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533c1a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0545d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c206599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,442 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:44:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:44:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a546036 2024-11-13T19:44:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:44:09,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:44:09,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2dccd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642b39c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae6a28a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,449 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd3d4c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,450 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247e7481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fe0364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,457 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662412b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,459 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3529fb5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a5a0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,463 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153c4687, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db3ad10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52267517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34817763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6f58ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4b43fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40564dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb12aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa78cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,472 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cbf99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,473 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b40e651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06ee61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,476 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ef581e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b2e532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31e52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:09,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:09,481 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b77971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:09,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375173c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:09,490 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,491 DEBUG [Thread-1132 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,491 DEBUG [Thread-1132 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,494 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,494 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,498 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,498 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,498 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,499 DEBUG [Thread-1147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,500 DEBUG [Thread-1147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,501 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,502 DEBUG [Thread-1145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,503 DEBUG [Thread-1145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,505 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,505 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,507 DEBUG [Thread-1136 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,507 DEBUG [Thread-1136 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:09,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,509 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:09,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:09,512 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:09,513 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:09,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:09,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:44:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527049538, max timestamp=1731527049548Num uniques 859; 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527049538, max timestamp=1731527049548Num uniques 859; 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527049538, max timestamp=1731527049548Num uniques 859; 2024-11-13T19:44:09,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:09,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/0a83b223662d454cbf9d0ee2e999b101 is 50, key is test_row_0/A:col10/1731527049508/Put/seqid=0 2024-11-13T19:44:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527109589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742068_1244 (size=12001) 2024-11-13T19:44:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527109589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/0a83b223662d454cbf9d0ee2e999b101 2024-11-13T19:44:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527109590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527109591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527109594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:44:09,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f41627da5eb347d0b576a7bdff16bd43 is 50, key is test_row_0/B:col10/1731527049508/Put/seqid=0 2024-11-13T19:44:09,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742069_1245 (size=12001) 2024-11-13T19:44:09,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f41627da5eb347d0b576a7bdff16bd43 2024-11-13T19:44:09,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:44:09,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:09,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:09,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:09,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:09,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:09,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/614f6b707c744588aeaaeee6fcf7066f is 50, key is test_row_0/C:col10/1731527049508/Put/seqid=0 2024-11-13T19:44:09,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742070_1246 (size=12001) 2024-11-13T19:44:09,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/614f6b707c744588aeaaeee6fcf7066f 2024-11-13T19:44:09,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/0a83b223662d454cbf9d0ee2e999b101 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101 2024-11-13T19:44:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527109714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527109714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527109715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527109714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527109715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:44:09,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f41627da5eb347d0b576a7bdff16bd43 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43 2024-11-13T19:44:09,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:44:09,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/614f6b707c744588aeaaeee6fcf7066f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f 2024-11-13T19:44:09,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:44:09,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6f4ececb48a6d1da14a168d390e345dd in 241ms, sequenceid=12, compaction requested=false 2024-11-13T19:44:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:44:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:09,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:44:09,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:09,831 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:44:09,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1d0ffa5e81a8426597dc05dd57777fa2 is 50, key is test_row_0/A:col10/1731527049586/Put/seqid=0 2024-11-13T19:44:09,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742071_1247 (size=12001) 2024-11-13T19:44:09,880 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1d0ffa5e81a8426597dc05dd57777fa2 2024-11-13T19:44:09,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9c3e6a571a484942859accae4f0438c5 is 50, key is test_row_0/B:col10/1731527049586/Put/seqid=0 2024-11-13T19:44:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:09,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742072_1248 (size=12001) 2024-11-13T19:44:09,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527109970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527109971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527109974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527109974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527109978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:09,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527110084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527110084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527110094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527110094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527110097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:44:10,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527110294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527110297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527110303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527110314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527110317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,378 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9c3e6a571a484942859accae4f0438c5 2024-11-13T19:44:10,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c612f240e84b4307aecf860b14afb6e0 is 50, key is test_row_0/C:col10/1731527049586/Put/seqid=0 2024-11-13T19:44:10,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742073_1249 (size=12001) 2024-11-13T19:44:10,444 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c612f240e84b4307aecf860b14afb6e0 2024-11-13T19:44:10,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1d0ffa5e81a8426597dc05dd57777fa2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2 2024-11-13T19:44:10,463 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T19:44:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9c3e6a571a484942859accae4f0438c5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5 2024-11-13T19:44:10,489 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T19:44:10,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c612f240e84b4307aecf860b14afb6e0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0 2024-11-13T19:44:10,500 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T19:44:10,503 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 6f4ececb48a6d1da14a168d390e345dd in 673ms, sequenceid=47, compaction requested=false 2024-11-13T19:44:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T19:44:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T19:44:10,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T19:44:10,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 994 msec 2024-11-13T19:44:10,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.0010 sec 2024-11-13T19:44:10,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:10,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/38f11ff2687542118d3b5f6d22f9848c is 50, key is test_row_0/A:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:10,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742074_1250 (size=12001) 2024-11-13T19:44:10,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/38f11ff2687542118d3b5f6d22f9848c 2024-11-13T19:44:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:44:10,653 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:10,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T19:44:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:44:10,656 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:10,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:10,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:10,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527110662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527110662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527110669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527110673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9b02e3a4b3df4ee0aa8afb4db376fc0b is 50, key is test_row_0/B:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527110682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742075_1251 (size=12001) 2024-11-13T19:44:10,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9b02e3a4b3df4ee0aa8afb4db376fc0b 2024-11-13T19:44:10,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/631efde143fa4946a4ffa11bd28f704f is 50, key is test_row_0/C:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:44:10,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742076_1252 (size=12001) 2024-11-13T19:44:10,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/631efde143fa4946a4ffa11bd28f704f 2024-11-13T19:44:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527110783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527110784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527110784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527110795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527110797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:10,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:44:10,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:10,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:10,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:10,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/38f11ff2687542118d3b5f6d22f9848c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c 2024-11-13T19:44:10,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-13T19:44:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/9b02e3a4b3df4ee0aa8afb4db376fc0b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b 2024-11-13T19:44:10,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-13T19:44:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/631efde143fa4946a4ffa11bd28f704f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f 2024-11-13T19:44:10,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-13T19:44:10,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6f4ececb48a6d1da14a168d390e345dd in 245ms, sequenceid=60, compaction requested=true 2024-11-13T19:44:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:10,864 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:10,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:10,865 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:10,865 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:10,865 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,865 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.2 K 2024-11-13T19:44:10,866 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a83b223662d454cbf9d0ee2e999b101, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731527049504 2024-11-13T19:44:10,866 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0ffa5e81a8426597dc05dd57777fa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731527049586 2024-11-13T19:44:10,867 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f11ff2687542118d3b5f6d22f9848c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:10,870 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:10,870 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:10,870 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,870 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.2 K 2024-11-13T19:44:10,871 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41627da5eb347d0b576a7bdff16bd43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731527049504 2024-11-13T19:44:10,874 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3e6a571a484942859accae4f0438c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731527049586 2024-11-13T19:44:10,878 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b02e3a4b3df4ee0aa8afb4db376fc0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:10,896 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:10,896 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7d34ee60dab84389a08a15ff373f8fb5 is 50, key is test_row_0/A:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:10,897 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:10,898 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f393f3677e4245ca932858c8f7be05ae is 50, key is test_row_0/B:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742077_1253 (size=12104) 2024-11-13T19:44:10,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742078_1254 (size=12104) 2024-11-13T19:44:10,954 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7d34ee60dab84389a08a15ff373f8fb5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7d34ee60dab84389a08a15ff373f8fb5 2024-11-13T19:44:10,961 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f393f3677e4245ca932858c8f7be05ae as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f393f3677e4245ca932858c8f7be05ae 2024-11-13T19:44:10,967 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 7d34ee60dab84389a08a15ff373f8fb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:10,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:10,967 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527050864; duration=0sec 2024-11-13T19:44:10,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:10,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:10,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:10,971 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:10,971 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:10,971 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,971 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.2 K 2024-11-13T19:44:10,973 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 614f6b707c744588aeaaeee6fcf7066f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731527049504 2024-11-13T19:44:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:44:10,973 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c612f240e84b4307aecf860b14afb6e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731527049586 2024-11-13T19:44:10,975 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into f393f3677e4245ca932858c8f7be05ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:10,975 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:10,975 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527050865; duration=0sec 2024-11-13T19:44:10,975 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:10,975 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:10,976 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 631efde143fa4946a4ffa11bd28f704f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:10,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:10,978 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/d8053b73790f410d9727dd223aff68b9 is 50, key is test_row_0/A:col10/1731527050667/Put/seqid=0 2024-11-13T19:44:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:10,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:11,001 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:11,001 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/6e1376a6c6cb449d85a339e9b487cc3c is 50, key is test_row_0/C:col10/1731527049937/Put/seqid=0 2024-11-13T19:44:11,010 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:44:11,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742079_1255 (size=12001) 2024-11-13T19:44:11,018 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/d8053b73790f410d9727dd223aff68b9 2024-11-13T19:44:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527111015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527111017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527111018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527111020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527111021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742080_1256 (size=12104) 2024-11-13T19:44:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e3132b28874a42b1beeb4c15603c5800 is 50, key is test_row_0/B:col10/1731527050667/Put/seqid=0 2024-11-13T19:44:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742081_1257 (size=12001) 2024-11-13T19:44:11,077 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e3132b28874a42b1beeb4c15603c5800 2024-11-13T19:44:11,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9eb89100c2e744e9b00502a283620e43 is 50, key is test_row_0/C:col10/1731527050667/Put/seqid=0 2024-11-13T19:44:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742082_1258 (size=12001) 2024-11-13T19:44:11,107 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9eb89100c2e744e9b00502a283620e43 2024-11-13T19:44:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/d8053b73790f410d9727dd223aff68b9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9 2024-11-13T19:44:11,123 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T19:44:11,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e3132b28874a42b1beeb4c15603c5800 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800 2024-11-13T19:44:11,129 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T19:44:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9eb89100c2e744e9b00502a283620e43 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43 2024-11-13T19:44:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527111134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527111134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527111134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,138 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,139 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6f4ececb48a6d1da14a168d390e345dd in 161ms, sequenceid=81, compaction requested=false 2024-11-13T19:44:11,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:11,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T19:44:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T19:44:11,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T19:44:11,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 484 msec 2024-11-13T19:44:11,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:11,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:11,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 491 msec 2024-11-13T19:44:11,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/fa63ecf3cad34987a169a5daa01f8d33 is 50, key is test_row_0/A:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742083_1259 (size=14341) 2024-11-13T19:44:11,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/fa63ecf3cad34987a169a5daa01f8d33 2024-11-13T19:44:11,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e8bbbfc01161487f964895713464489a is 50, key is test_row_0/B:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527111247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:44:11,284 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:11,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T19:44:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:11,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:11,290 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:11,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:11,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742084_1260 (size=12001) 2024-11-13T19:44:11,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527111324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527111343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527111343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527111343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527111364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:11,442 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/6e1376a6c6cb449d85a339e9b487cc3c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6e1376a6c6cb449d85a339e9b487cc3c 2024-11-13T19:44:11,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T19:44:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,450 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 6e1376a6c6cb449d85a339e9b487cc3c(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:11,450 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:11,450 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=13, startTime=1731527050865; duration=0sec 2024-11-13T19:44:11,451 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:11,451 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527111574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:11,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T19:44:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527111654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527111655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527111655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e8bbbfc01161487f964895713464489a 2024-11-13T19:44:11,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/943270e5e1ce4aed9070cd46b55b3595 is 50, key is test_row_0/C:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742085_1261 (size=12001) 2024-11-13T19:44:11,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/943270e5e1ce4aed9070cd46b55b3595 2024-11-13T19:44:11,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/fa63ecf3cad34987a169a5daa01f8d33 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33 2024-11-13T19:44:11,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33, entries=200, sequenceid=95, filesize=14.0 K 2024-11-13T19:44:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e8bbbfc01161487f964895713464489a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a 2024-11-13T19:44:11,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T19:44:11,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/943270e5e1ce4aed9070cd46b55b3595 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595 2024-11-13T19:44:11,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T19:44:11,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6f4ececb48a6d1da14a168d390e345dd in 617ms, sequenceid=95, compaction requested=true 2024-11-13T19:44:11,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:11,762 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:11,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:11,762 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:11,763 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:11,763 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:11,763 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:11,763 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:11,763 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,763 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,763 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7d34ee60dab84389a08a15ff373f8fb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.5 K 2024-11-13T19:44:11,763 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f393f3677e4245ca932858c8f7be05ae, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.3 K 2024-11-13T19:44:11,765 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f393f3677e4245ca932858c8f7be05ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:11,765 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d34ee60dab84389a08a15ff373f8fb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:11,765 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8053b73790f410d9727dd223aff68b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731527050660 2024-11-13T19:44:11,765 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e3132b28874a42b1beeb4c15603c5800, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731527050660 2024-11-13T19:44:11,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T19:44:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,766 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:44:11,766 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bbbfc01161487f964895713464489a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:11,767 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa63ecf3cad34987a169a5daa01f8d33, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f is 50, key is test_row_0/A:col10/1731527051244/Put/seqid=0 2024-11-13T19:44:11,787 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:11,787 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/430d75f7a87c4466ab2809eeeed24f4e is 50, key is test_row_0/A:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,799 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:11,800 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/04ce271f088a4084b03c9ba95be2b68c is 50, key is test_row_0/B:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:11,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:11,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742086_1262 (size=12001) 2024-11-13T19:44:11,842 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f 2024-11-13T19:44:11,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742087_1263 (size=12207) 2024-11-13T19:44:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742088_1264 (size=12207) 2024-11-13T19:44:11,865 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/430d75f7a87c4466ab2809eeeed24f4e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/430d75f7a87c4466ab2809eeeed24f4e 2024-11-13T19:44:11,866 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/04ce271f088a4084b03c9ba95be2b68c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/04ce271f088a4084b03c9ba95be2b68c 2024-11-13T19:44:11,874 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 430d75f7a87c4466ab2809eeeed24f4e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:11,874 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:11,874 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527051762; duration=0sec 2024-11-13T19:44:11,874 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:11,874 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:11,874 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:11,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/bf02c9f53ea34c2dac74944fc9f7922e is 50, key is test_row_0/B:col10/1731527051244/Put/seqid=0 2024-11-13T19:44:11,875 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into 04ce271f088a4084b03c9ba95be2b68c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:11,876 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:11,876 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527051762; duration=0sec 2024-11-13T19:44:11,876 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:11,876 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:11,876 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:11,877 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:11,877 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:11,877 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6e1376a6c6cb449d85a339e9b487cc3c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.3 K 2024-11-13T19:44:11,878 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1376a6c6cb449d85a339e9b487cc3c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731527049937 2024-11-13T19:44:11,878 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb89100c2e744e9b00502a283620e43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731527050660 2024-11-13T19:44:11,879 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943270e5e1ce4aed9070cd46b55b3595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527111894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527111895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:11,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742089_1265 (size=12001) 2024-11-13T19:44:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:11,924 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:11,925 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/27cc8a94e4904c65a4637531597801fc is 50, key is test_row_0/C:col10/1731527051014/Put/seqid=0 2024-11-13T19:44:11,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742090_1266 (size=12207) 2024-11-13T19:44:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527112003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527112013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527112164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527112164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527112173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527112214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527112223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,313 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/bf02c9f53ea34c2dac74944fc9f7922e 2024-11-13T19:44:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cc92ca17a4464431980d6044802e7084 is 50, key is test_row_0/C:col10/1731527051244/Put/seqid=0 2024-11-13T19:44:12,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742091_1267 (size=12001) 2024-11-13T19:44:12,364 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/27cc8a94e4904c65a4637531597801fc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/27cc8a94e4904c65a4637531597801fc 2024-11-13T19:44:12,371 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 27cc8a94e4904c65a4637531597801fc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:12,371 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:12,372 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=13, startTime=1731527051762; duration=0sec 2024-11-13T19:44:12,372 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:12,372 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527112524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527112533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:12,741 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cc92ca17a4464431980d6044802e7084 2024-11-13T19:44:12,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f 2024-11-13T19:44:12,754 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:44:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/bf02c9f53ea34c2dac74944fc9f7922e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e 2024-11-13T19:44:12,760 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:44:12,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cc92ca17a4464431980d6044802e7084 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084 2024-11-13T19:44:12,767 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:44:12,768 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6f4ececb48a6d1da14a168d390e345dd in 1002ms, sequenceid=114, compaction requested=false 2024-11-13T19:44:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T19:44:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T19:44:12,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T19:44:12,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4800 sec 2024-11-13T19:44:12,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4890 sec 2024-11-13T19:44:13,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:13,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:13,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/bfe7781088b3465b891f293b9b557c1b is 50, key is test_row_0/A:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742092_1268 (size=14441) 2024-11-13T19:44:13,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527113100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527113101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527113175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,184 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527113185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527113185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,193 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,194 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527113424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527113424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:44:13,444 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:13,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T19:44:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T19:44:13,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:13,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:13,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:13,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/bfe7781088b3465b891f293b9b557c1b 2024-11-13T19:44:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T19:44:13,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/882f33527e7746b8bac23b46768c6e5d is 50, key is test_row_0/B:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742093_1269 (size=12051) 2024-11-13T19:44:13,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/882f33527e7746b8bac23b46768c6e5d 2024-11-13T19:44:13,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T19:44:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/58835069416941afac73754b55e8970b is 50, key is test_row_0/C:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742094_1270 (size=12051) 2024-11-13T19:44:13,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/58835069416941afac73754b55e8970b 2024-11-13T19:44:13,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/bfe7781088b3465b891f293b9b557c1b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b 2024-11-13T19:44:13,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b, entries=200, sequenceid=132, filesize=14.1 K 2024-11-13T19:44:13,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/882f33527e7746b8bac23b46768c6e5d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d 2024-11-13T19:44:13,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527113737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527113737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:13,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T19:44:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/58835069416941afac73754b55e8970b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b 2024-11-13T19:44:13,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T19:44:13,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T19:44:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:13,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6f4ececb48a6d1da14a168d390e345dd in 740ms, sequenceid=132, compaction requested=true 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:13,782 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:13,782 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T19:44:13,785 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:13,785 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:13,786 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,786 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/04ce271f088a4084b03c9ba95be2b68c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.4 K 2024-11-13T19:44:13,786 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:13,786 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:13,786 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,786 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/430d75f7a87c4466ab2809eeeed24f4e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.7 K 2024-11-13T19:44:13,786 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 430d75f7a87c4466ab2809eeeed24f4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:13,786 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ce271f088a4084b03c9ba95be2b68c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:13,787 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf02c9f53ea34c2dac74944fc9f7922e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731527051207 2024-11-13T19:44:13,787 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882f33527e7746b8bac23b46768c6e5d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:13,796 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:13,796 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting abfa5e3fae7046b2a88f1e1a6fd1ff9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731527051207 2024-11-13T19:44:13,797 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/2589fa09e3fe43f6b702b5ce0d8e3294 is 50, key is test_row_0/B:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,797 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe7781088b3465b891f293b9b557c1b, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742095_1271 (size=12359) 2024-11-13T19:44:13,816 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:13,816 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c8a31d4c34a74194a148a6738c5f76f7 is 50, key is test_row_0/A:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742096_1272 (size=12359) 2024-11-13T19:44:13,847 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c8a31d4c34a74194a148a6738c5f76f7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c8a31d4c34a74194a148a6738c5f76f7 2024-11-13T19:44:13,855 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into c8a31d4c34a74194a148a6738c5f76f7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:13,855 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:13,855 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527053782; duration=0sec 2024-11-13T19:44:13,855 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:13,855 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:13,855 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:13,858 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:13,858 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:13,858 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,858 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/27cc8a94e4904c65a4637531597801fc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=35.4 K 2024-11-13T19:44:13,859 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 27cc8a94e4904c65a4637531597801fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731527051014 2024-11-13T19:44:13,859 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cc92ca17a4464431980d6044802e7084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731527051207 2024-11-13T19:44:13,860 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 58835069416941afac73754b55e8970b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:13,883 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#227 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:13,884 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9e65d88a4d4d4519aaed69e336d17a31 is 50, key is test_row_0/C:col10/1731527051840/Put/seqid=0 2024-11-13T19:44:13,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742097_1273 (size=12359) 2024-11-13T19:44:13,911 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9e65d88a4d4d4519aaed69e336d17a31 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9e65d88a4d4d4519aaed69e336d17a31 2024-11-13T19:44:13,920 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 9e65d88a4d4d4519aaed69e336d17a31(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:13,920 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:13,920 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=13, startTime=1731527053782; duration=0sec 2024-11-13T19:44:13,920 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:13,920 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:13,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T19:44:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:13,934 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/739f128ab23b4edb8ab847d1552ca135 is 50, key is test_row_0/A:col10/1731527053086/Put/seqid=0 2024-11-13T19:44:13,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742098_1274 (size=12151) 2024-11-13T19:44:13,954 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/739f128ab23b4edb8ab847d1552ca135 2024-11-13T19:44:13,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/1952393b96f54adc9c9d50637f840972 is 50, key is test_row_0/B:col10/1731527053086/Put/seqid=0 2024-11-13T19:44:13,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742099_1275 (size=12151) 2024-11-13T19:44:13,988 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/1952393b96f54adc9c9d50637f840972 2024-11-13T19:44:14,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/8d146be8db934dc087ef94e455453ab2 is 50, key is test_row_0/C:col10/1731527053086/Put/seqid=0 2024-11-13T19:44:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742100_1276 (size=12151) 2024-11-13T19:44:14,040 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/8d146be8db934dc087ef94e455453ab2 2024-11-13T19:44:14,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/739f128ab23b4edb8ab847d1552ca135 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135 2024-11-13T19:44:14,056 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:44:14,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/1952393b96f54adc9c9d50637f840972 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972 2024-11-13T19:44:14,062 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:44:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/8d146be8db934dc087ef94e455453ab2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2 2024-11-13T19:44:14,069 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:44:14,071 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 6f4ececb48a6d1da14a168d390e345dd in 136ms, sequenceid=151, compaction requested=false 2024-11-13T19:44:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T19:44:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T19:44:14,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T19:44:14,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-13T19:44:14,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 631 msec 2024-11-13T19:44:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T19:44:14,094 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:14,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-13T19:44:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:14,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:14,102 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:14,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:14,211 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/2589fa09e3fe43f6b702b5ce0d8e3294 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2589fa09e3fe43f6b702b5ce0d8e3294 2024-11-13T19:44:14,217 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into 2589fa09e3fe43f6b702b5ce0d8e3294(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:14,217 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:14,217 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527053782; duration=0sec 2024-11-13T19:44:14,218 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:14,218 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:14,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T19:44:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:14,264 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:14,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9cf743becd5b44e7b0c1a5e028d92521 is 50, key is test_row_2/A:col10/1731527054254/Put/seqid=0 2024-11-13T19:44:14,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742101_1277 (size=7365) 2024-11-13T19:44:14,294 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9cf743becd5b44e7b0c1a5e028d92521 2024-11-13T19:44:14,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c5220ab666da4bf4a0b625441caa7fc1 is 50, key is test_row_2/B:col10/1731527054254/Put/seqid=0 2024-11-13T19:44:14,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527054321, max timestamp=1731527054332Num uniques 859; 2024-11-13T19:44:14,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:14,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527054321, max timestamp=1731527054332Num uniques 859; 2024-11-13T19:44:14,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:44:14,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527054321, max timestamp=1731527054332Num uniques 859; 2024-11-13T19:44:14,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:44:14,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742102_1278 (size=7365) 2024-11-13T19:44:14,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527054379, max timestamp=1731527054389Num uniques 1018; 2024-11-13T19:44:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:44:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527054379, max timestamp=1731527054389Num uniques 1009; 2024-11-13T19:44:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-13T19:44:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527054379, max timestamp=1731527054389Num uniques 1009; 2024-11-13T19:44:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-13T19:44:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:14,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1731527054413, max timestamp=1731527054451Num uniques 1077; 2024-11-13T19:44:14,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=8092, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:44:14,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1731527054413, max timestamp=1731527054451Num uniques 1077; 2024-11-13T19:44:14,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=8092, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:44:14,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1731527054413, max timestamp=1731527054451Num uniques 1077; 2024-11-13T19:44:14,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=8092, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:44:14,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527054486, max timestamp=1731527054501Num uniques 1036; 2024-11-13T19:44:14,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:44:14,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527054486, max timestamp=1731527054501Num uniques 1036; 2024-11-13T19:44:14,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:44:14,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527054486, max timestamp=1731527054501Num uniques 1036; 2024-11-13T19:44:14,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:44:14,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731527054529, max timestamp=1731527054552Num uniques 1045; 2024-11-13T19:44:14,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-13T19:44:14,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731527054529, max timestamp=1731527054552Num uniques 1045; 2024-11-13T19:44:14,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-13T19:44:14,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731527054529, max timestamp=1731527054552Num uniques 1045; 2024-11-13T19:44:14,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-13T19:44:14,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527054592, max timestamp=1731527054613Num uniques 1104; 2024-11-13T19:44:14,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527054592, max timestamp=1731527054613Num uniques 1104; 2024-11-13T19:44:14,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-13T19:44:14,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-13T19:44:14,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527054592, max timestamp=1731527054613Num uniques 1104; 2024-11-13T19:44:14,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-13T19:44:14,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731527054656, max timestamp=1731527054682Num uniques 1013; 2024-11-13T19:44:14,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-13T19:44:14,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731527054656, max timestamp=1731527054682Num uniques 1013; 2024-11-13T19:44:14,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-13T19:44:14,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731527054656, max timestamp=1731527054682Num uniques 1013; 2024-11-13T19:44:14,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-13T19:44:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:14,749 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c5220ab666da4bf4a0b625441caa7fc1 2024-11-13T19:44:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/4b6772f014aa4ff2b019e4d5ad2236ac is 50, key is test_row_2/C:col10/1731527054254/Put/seqid=0 2024-11-13T19:44:14,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527054723, max timestamp=1731527054795Num uniques 1022; 2024-11-13T19:44:14,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-13T19:44:14,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527054723, max timestamp=1731527054795Num uniques 1022; 2024-11-13T19:44:14,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-13T19:44:14,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527054723, max timestamp=1731527054795Num uniques 1022; 2024-11-13T19:44:14,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-13T19:44:14,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742103_1279 (size=7365) 2024-11-13T19:44:14,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731527054862, max timestamp=1731527054878Num uniques 1031; 2024-11-13T19:44:14,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-13T19:44:14,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731527054862, max timestamp=1731527054878Num uniques 1031; 2024-11-13T19:44:14,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-13T19:44:14,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731527054862, max timestamp=1731527054878Num uniques 1031; 2024-11-13T19:44:14,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-13T19:44:14,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731527054916, max timestamp=1731527054949Num uniques 1040; 2024-11-13T19:44:14,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-13T19:44:14,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1731527054916, max timestamp=1731527054949Num uniques 1040; 2024-11-13T19:44:14,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47620, new segment data size=8690, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-13T19:44:14,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8691, totalHeapSize=20000, min timestamp=1731527054916, max timestamp=1731527054949Num uniques 1040; 2024-11-13T19:44:14,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47621, new segment data size=8691, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731527054977, max timestamp=1731527055004Num uniques 1049; 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731527054977, max timestamp=1731527055004Num uniques 1049; 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731527054977, max timestamp=1731527055004Num uniques 1049; 2024-11-13T19:44:15,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-13T19:44:15,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527055044, max timestamp=1731527055056Num uniques 1108; 2024-11-13T19:44:15,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T19:44:15,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527055044, max timestamp=1731527055056Num uniques 1108; 2024-11-13T19:44:15,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T19:44:15,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527055044, max timestamp=1731527055056Num uniques 1108; 2024-11-13T19:44:15,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-13T19:44:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527055079, max timestamp=1731527055103Num uniques 1017; 2024-11-13T19:44:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T19:44:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527055079, max timestamp=1731527055103Num uniques 1017; 2024-11-13T19:44:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T19:44:15,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527055079, max timestamp=1731527055103Num uniques 1017; 2024-11-13T19:44:15,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-13T19:44:15,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731527055123, max timestamp=1731527055153Num uniques 1026; 2024-11-13T19:44:15,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T19:44:15,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731527055123, max timestamp=1731527055153Num uniques 1026; 2024-11-13T19:44:15,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T19:44:15,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731527055123, max timestamp=1731527055153Num uniques 1026; 2024-11-13T19:44:15,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-13T19:44:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731527055176, max timestamp=1731527055204Num uniques 1085; 2024-11-13T19:44:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-13T19:44:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731527055176, max timestamp=1731527055204Num uniques 1085; 2024-11-13T19:44:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-13T19:44:15,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731527055176, max timestamp=1731527055204Num uniques 1085; 2024-11-13T19:44:15,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-13T19:44:15,215 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/4b6772f014aa4ff2b019e4d5ad2236ac 2024-11-13T19:44:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9cf743becd5b44e7b0c1a5e028d92521 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521 2024-11-13T19:44:15,227 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521, entries=50, sequenceid=156, filesize=7.2 K 2024-11-13T19:44:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c5220ab666da4bf4a0b625441caa7fc1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1 2024-11-13T19:44:15,232 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1, entries=50, sequenceid=156, filesize=7.2 K 2024-11-13T19:44:15,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731527055222, max timestamp=1731527055230Num uniques 1044; 2024-11-13T19:44:15,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-13T19:44:15,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731527055222, max timestamp=1731527055230Num uniques 1044; 2024-11-13T19:44:15,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-13T19:44:15,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731527055222, max timestamp=1731527055230Num uniques 1044; 2024-11-13T19:44:15,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-13T19:44:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:15,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/4b6772f014aa4ff2b019e4d5ad2236ac as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac 2024-11-13T19:44:15,245 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac, entries=50, sequenceid=156, filesize=7.2 K 2024-11-13T19:44:15,246 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=67.09 KB/68700 for 6f4ececb48a6d1da14a168d390e345dd in 990ms, sequenceid=156, compaction requested=true 2024-11-13T19:44:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-13T19:44:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:15,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=80.51 KB heapSize=217.64 KB 2024-11-13T19:44:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:15,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T19:44:15,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1450 sec 2024-11-13T19:44:15,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.1560 sec 2024-11-13T19:44:15,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1dc68c6661434061aafa69c8c10dee89 is 50, key is test_row_0/A:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742104_1280 (size=12301) 2024-11-13T19:44:15,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1dc68c6661434061aafa69c8c10dee89 2024-11-13T19:44:15,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527115275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527115276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527115282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527115283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527115285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/70b32dc2d1d441ca92be9ed9a3a794c1 is 50, key is test_row_0/B:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742105_1281 (size=12301) 2024-11-13T19:44:15,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/70b32dc2d1d441ca92be9ed9a3a794c1 2024-11-13T19:44:15,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/03448221d9714e21bdae2f05b89db39d is 50, key is test_row_0/C:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742106_1282 (size=12301) 2024-11-13T19:44:15,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/03448221d9714e21bdae2f05b89db39d 2024-11-13T19:44:15,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1dc68c6661434061aafa69c8c10dee89 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89 2024-11-13T19:44:15,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527115394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527115394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527115394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527115394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89, entries=150, sequenceid=442, filesize=12.0 K 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527115394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/70b32dc2d1d441ca92be9ed9a3a794c1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1 2024-11-13T19:44:15,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1, entries=150, sequenceid=442, filesize=12.0 K 2024-11-13T19:44:15,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/03448221d9714e21bdae2f05b89db39d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d 2024-11-13T19:44:15,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d, entries=150, sequenceid=442, filesize=12.0 K 2024-11-13T19:44:15,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~237.52 KB/243216, currentSize=93.93 KB/96180 for 6f4ececb48a6d1da14a168d390e345dd in 171ms, sequenceid=442, compaction requested=true 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:15,417 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:15,417 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:15,419 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44176 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:15,419 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:15,419 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:15,419 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c8a31d4c34a74194a148a6738c5f76f7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=43.1 K 2024-11-13T19:44:15,419 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44176 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:15,419 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:15,419 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:15,419 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2589fa09e3fe43f6b702b5ce0d8e3294, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=43.1 K 2024-11-13T19:44:15,420 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a31d4c34a74194a148a6738c5f76f7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:15,420 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2589fa09e3fe43f6b702b5ce0d8e3294, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:15,421 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739f128ab23b4edb8ab847d1552ca135, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731527053081 2024-11-13T19:44:15,421 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1952393b96f54adc9c9d50637f840972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731527053081 2024-11-13T19:44:15,422 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf743becd5b44e7b0c1a5e028d92521, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731527054254 2024-11-13T19:44:15,422 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c5220ab666da4bf4a0b625441caa7fc1, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731527054254 2024-11-13T19:44:15,423 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b32dc2d1d441ca92be9ed9a3a794c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:15,423 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc68c6661434061aafa69c8c10dee89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:15,436 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:15,437 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f4c3ebde605d4a1da640d04313e6aecb is 50, key is test_row_0/B:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,440 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:15,440 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/54882177d1524c3dac16d5c38cae0d88 is 50, key is test_row_0/A:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742107_1283 (size=12745) 2024-11-13T19:44:15,465 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/f4c3ebde605d4a1da640d04313e6aecb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f4c3ebde605d4a1da640d04313e6aecb 2024-11-13T19:44:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742108_1284 (size=12745) 2024-11-13T19:44:15,479 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into f4c3ebde605d4a1da640d04313e6aecb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:15,479 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:15,479 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=12, startTime=1731527055417; duration=0sec 2024-11-13T19:44:15,479 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:15,479 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:15,479 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:15,481 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44176 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:15,481 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:15,481 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:15,481 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9e65d88a4d4d4519aaed69e336d17a31, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=43.1 K 2024-11-13T19:44:15,485 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e65d88a4d4d4519aaed69e336d17a31, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731527051840 2024-11-13T19:44:15,491 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d146be8db934dc087ef94e455453ab2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731527053081 2024-11-13T19:44:15,492 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/54882177d1524c3dac16d5c38cae0d88 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/54882177d1524c3dac16d5c38cae0d88 2024-11-13T19:44:15,492 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6772f014aa4ff2b019e4d5ad2236ac, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731527054254 2024-11-13T19:44:15,493 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 03448221d9714e21bdae2f05b89db39d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:15,500 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 54882177d1524c3dac16d5c38cae0d88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:15,500 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:15,500 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=12, startTime=1731527055417; duration=0sec 2024-11-13T19:44:15,500 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:15,500 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:15,507 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:15,508 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/b1024479182c4adb9ae4d89a065cd4cc is 50, key is test_row_0/C:col10/1731527055222/Put/seqid=0 2024-11-13T19:44:15,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742109_1285 (size=12745) 2024-11-13T19:44:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:15,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:15,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:15,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/aad3a2bdf09b4cc890cc631e623cf56d is 50, key is test_row_0/A:col10/1731527055609/Put/seqid=0 2024-11-13T19:44:15,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742110_1286 (size=12301) 2024-11-13T19:44:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527115624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527115631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527115632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527115631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527115632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527115744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527115744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527115745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527115745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527115745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527115954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527115954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527115954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527115954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527115954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,033 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/b1024479182c4adb9ae4d89a065cd4cc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b1024479182c4adb9ae4d89a065cd4cc 2024-11-13T19:44:16,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/aad3a2bdf09b4cc890cc631e623cf56d 2024-11-13T19:44:16,057 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into b1024479182c4adb9ae4d89a065cd4cc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:16,057 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:16,057 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=12, startTime=1731527055417; duration=0sec 2024-11-13T19:44:16,057 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:16,057 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:16,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/40d1d547800f4aa59cc943371663cf80 is 50, key is test_row_0/B:col10/1731527055609/Put/seqid=0 2024-11-13T19:44:16,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742111_1287 (size=12301) 2024-11-13T19:44:16,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/40d1d547800f4aa59cc943371663cf80 2024-11-13T19:44:16,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/54da8f98fe544d0da20fa0711cfbc3fa is 50, key is test_row_0/C:col10/1731527055609/Put/seqid=0 2024-11-13T19:44:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742112_1288 (size=12301) 2024-11-13T19:44:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:44:16,254 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:16,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T19:44:16,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:16,258 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:16,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527116264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527116264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527116264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527116265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527116277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:16,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T19:44:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:16,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/54da8f98fe544d0da20fa0711cfbc3fa 2024-11-13T19:44:16,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T19:44:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:16,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/aad3a2bdf09b4cc890cc631e623cf56d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d 2024-11-13T19:44:16,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d, entries=150, sequenceid=463, filesize=12.0 K 2024-11-13T19:44:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/40d1d547800f4aa59cc943371663cf80 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80 2024-11-13T19:44:16,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80, entries=150, sequenceid=463, filesize=12.0 K 2024-11-13T19:44:16,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/54da8f98fe544d0da20fa0711cfbc3fa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa 2024-11-13T19:44:16,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa, entries=150, sequenceid=463, filesize=12.0 K 2024-11-13T19:44:16,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 6f4ececb48a6d1da14a168d390e345dd in 994ms, sequenceid=463, compaction requested=false 2024-11-13T19:44:16,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:16,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T19:44:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:16,726 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:16,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/56735afe4e02475b877c219e9d9f8af0 is 50, key is test_row_0/A:col10/1731527055627/Put/seqid=0 2024-11-13T19:44:16,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742113_1289 (size=12301) 2024-11-13T19:44:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527116805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527116806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527116808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527116809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527116810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527116913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527116924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527116924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527116924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527116941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527117124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527117133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527117133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,137 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/56735afe4e02475b877c219e9d9f8af0 2024-11-13T19:44:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527117137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527117157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ef81fd1c1e584d7fbedac49460ac3777 is 50, key is test_row_0/B:col10/1731527055627/Put/seqid=0 2024-11-13T19:44:17,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742114_1290 (size=12301) 2024-11-13T19:44:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527117435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527117444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527117444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527117454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527117474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ef81fd1c1e584d7fbedac49460ac3777 2024-11-13T19:44:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/29f8ca2a7d36453d9af8a9c9beaad7ba is 50, key is test_row_0/C:col10/1731527055627/Put/seqid=0 2024-11-13T19:44:17,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742115_1291 (size=12301) 2024-11-13T19:44:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527117943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527117964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527117964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527117965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527117993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:18,034 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/29f8ca2a7d36453d9af8a9c9beaad7ba 2024-11-13T19:44:18,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/56735afe4e02475b877c219e9d9f8af0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0 2024-11-13T19:44:18,094 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0, entries=150, sequenceid=478, filesize=12.0 K 2024-11-13T19:44:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ef81fd1c1e584d7fbedac49460ac3777 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777 2024-11-13T19:44:18,100 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777, entries=150, sequenceid=478, filesize=12.0 K 2024-11-13T19:44:18,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/29f8ca2a7d36453d9af8a9c9beaad7ba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba 2024-11-13T19:44:18,107 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba, entries=150, sequenceid=478, filesize=12.0 K 2024-11-13T19:44:18,109 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6f4ececb48a6d1da14a168d390e345dd in 1384ms, sequenceid=478, compaction requested=true 2024-11-13T19:44:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T19:44:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T19:44:18,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T19:44:18,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8540 sec 2024-11-13T19:44:18,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8670 sec 2024-11-13T19:44:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T19:44:18,404 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:18,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T19:44:18,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:44:18,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:18,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:44:18,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:44:18,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:18,571 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:44:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8a369b80a41445dc88a66d109fbd3fa4 is 50, key is test_row_0/A:col10/1731527056807/Put/seqid=0 2024-11-13T19:44:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742116_1292 (size=12301) 2024-11-13T19:44:18,623 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8a369b80a41445dc88a66d109fbd3fa4 2024-11-13T19:44:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/fcc036ec4a3849928e8f585d80d60b7d is 50, key is test_row_0/B:col10/1731527056807/Put/seqid=0 2024-11-13T19:44:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742117_1293 (size=12301) 2024-11-13T19:44:18,698 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/fcc036ec4a3849928e8f585d80d60b7d 2024-11-13T19:44:18,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/06aea85c200d49bd93d5061ba532a0f9 is 50, key is test_row_0/C:col10/1731527056807/Put/seqid=0 2024-11-13T19:44:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:44:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742118_1294 (size=12301) 2024-11-13T19:44:18,735 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/06aea85c200d49bd93d5061ba532a0f9 2024-11-13T19:44:18,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8a369b80a41445dc88a66d109fbd3fa4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4 2024-11-13T19:44:18,773 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4, entries=150, sequenceid=496, filesize=12.0 K 2024-11-13T19:44:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/fcc036ec4a3849928e8f585d80d60b7d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d 2024-11-13T19:44:18,792 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d, entries=150, sequenceid=496, filesize=12.0 K 2024-11-13T19:44:18,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/06aea85c200d49bd93d5061ba532a0f9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9 2024-11-13T19:44:18,800 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9, entries=150, sequenceid=496, filesize=12.0 K 2024-11-13T19:44:18,801 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 6f4ececb48a6d1da14a168d390e345dd in 230ms, sequenceid=496, compaction requested=true 2024-11-13T19:44:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T19:44:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T19:44:18,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T19:44:18,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 394 msec 2024-11-13T19:44:18,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 400 msec 2024-11-13T19:44:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:18,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:19,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/096e10e75a53403fa7c2e9da5444f2cf is 50, key is test_row_0/A:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742119_1295 (size=9857) 2024-11-13T19:44:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:44:19,042 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:19,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/096e10e75a53403fa7c2e9da5444f2cf 2024-11-13T19:44:19,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T19:44:19,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c6e3a5574e9a4f75907dc4007d2534dc is 50, key is test_row_0/B:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:19,052 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:19,053 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:19,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:19,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742120_1296 (size=9857) 2024-11-13T19:44:19,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c6e3a5574e9a4f75907dc4007d2534dc 2024-11-13T19:44:19,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/6ba94d0d7e9944cc9511e83c8d88e0a5 is 50, key is test_row_0/C:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:19,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527059073, max timestamp=1731527059110Num uniques 859; 2024-11-13T19:44:19,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:19,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527059073, max timestamp=1731527059110Num uniques 859; 2024-11-13T19:44:19,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:19,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527059073, max timestamp=1731527059110Num uniques 859; 2024-11-13T19:44:19,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:19,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742121_1297 (size=9857) 2024-11-13T19:44:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:19,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527059137, max timestamp=1731527059171Num uniques 1018; 2024-11-13T19:44:19,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T19:44:19,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527059137, max timestamp=1731527059171Num uniques 1018; 2024-11-13T19:44:19,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T19:44:19,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527059137, max timestamp=1731527059171Num uniques 1018; 2024-11-13T19:44:19,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T19:44:19,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:44:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527119199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527119202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527119203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527119211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527119211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527119314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527119315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527119324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527119324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527119325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:44:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:19,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:44:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:19,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/6ba94d0d7e9944cc9511e83c8d88e0a5 2024-11-13T19:44:19,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527119524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527119524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/096e10e75a53403fa7c2e9da5444f2cf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf 2024-11-13T19:44:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527119534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527119534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf, entries=100, sequenceid=506, filesize=9.6 K 2024-11-13T19:44:19,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c6e3a5574e9a4f75907dc4007d2534dc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc 2024-11-13T19:44:19,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc, entries=100, sequenceid=506, filesize=9.6 K 2024-11-13T19:44:19,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/6ba94d0d7e9944cc9511e83c8d88e0a5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5 2024-11-13T19:44:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527119543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5, entries=100, sequenceid=506, filesize=9.6 K 2024-11-13T19:44:19,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6f4ececb48a6d1da14a168d390e345dd in 565ms, sequenceid=506, compaction requested=true 2024-11-13T19:44:19,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:19,559 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:19,560 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59505 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:19,560 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:19,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/54882177d1524c3dac16d5c38cae0d88, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=58.1 K 2024-11-13T19:44:19,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54882177d1524c3dac16d5c38cae0d88, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:19,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad3a2bdf09b4cc890cc631e623cf56d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731527055271 2024-11-13T19:44:19,562 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56735afe4e02475b877c219e9d9f8af0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731527055620 2024-11-13T19:44:19,562 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a369b80a41445dc88a66d109fbd3fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731527056803 2024-11-13T19:44:19,562 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 096e10e75a53403fa7c2e9da5444f2cf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527058986 2024-11-13T19:44:19,576 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#252 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:19,577 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/2cfe874205e647fe8f8862fb8126bb16 is 50, key is test_row_0/A:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:19,577 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:19,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:19,580 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59505 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:19,580 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:19,580 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,581 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f4c3ebde605d4a1da640d04313e6aecb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=58.1 K 2024-11-13T19:44:19,582 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c3ebde605d4a1da640d04313e6aecb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:19,584 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d1d547800f4aa59cc943371663cf80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731527055271 2024-11-13T19:44:19,585 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ef81fd1c1e584d7fbedac49460ac3777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731527055620 2024-11-13T19:44:19,585 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc036ec4a3849928e8f585d80d60b7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731527056803 2024-11-13T19:44:19,586 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e3a5574e9a4f75907dc4007d2534dc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527058986 2024-11-13T19:44:19,609 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#253 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:19,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742122_1298 (size=12915) 2024-11-13T19:44:19,610 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/0f8c4d6655374bb8a81b397b07e13bb0 is 50, key is test_row_0/B:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:19,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742123_1299 (size=12915) 2024-11-13T19:44:19,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:44:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:19,674 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c0e051c4203c488e9fbce77f72635f03 is 50, key is test_row_0/A:col10/1731527059196/Put/seqid=0 2024-11-13T19:44:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:19,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742124_1300 (size=12301) 2024-11-13T19:44:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c0e051c4203c488e9fbce77f72635f03 2024-11-13T19:44:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c8be12670010417bbd49aa97f5cee8a9 is 50, key is test_row_0/B:col10/1731527059196/Put/seqid=0 2024-11-13T19:44:19,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742125_1301 (size=12301) 2024-11-13T19:44:19,733 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c8be12670010417bbd49aa97f5cee8a9 2024-11-13T19:44:19,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/b56ee06a11b94b8591c850a609ae54c6 is 50, key is test_row_0/C:col10/1731527059196/Put/seqid=0 2024-11-13T19:44:19,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742126_1302 (size=12301) 2024-11-13T19:44:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:19,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:19,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527119852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527119853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527119853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527119859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527119861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527119973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527119973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527119973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527119974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,017 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/2cfe874205e647fe8f8862fb8126bb16 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/2cfe874205e647fe8f8862fb8126bb16 2024-11-13T19:44:20,023 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 2cfe874205e647fe8f8862fb8126bb16(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:20,023 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,023 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=11, startTime=1731527059558; duration=0sec 2024-11-13T19:44:20,023 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:20,023 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:20,023 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:20,026 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59505 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:20,026 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:20,027 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:20,027 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b1024479182c4adb9ae4d89a065cd4cc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=58.1 K 2024-11-13T19:44:20,028 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1024479182c4adb9ae4d89a065cd4cc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731527055222 2024-11-13T19:44:20,029 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54da8f98fe544d0da20fa0711cfbc3fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731527055271 2024-11-13T19:44:20,029 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29f8ca2a7d36453d9af8a9c9beaad7ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731527055620 2024-11-13T19:44:20,030 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06aea85c200d49bd93d5061ba532a0f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731527056803 2024-11-13T19:44:20,030 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba94d0d7e9944cc9511e83c8d88e0a5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527058986 2024-11-13T19:44:20,044 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/0f8c4d6655374bb8a81b397b07e13bb0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0f8c4d6655374bb8a81b397b07e13bb0 2024-11-13T19:44:20,050 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into 0f8c4d6655374bb8a81b397b07e13bb0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:20,050 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,050 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=11, startTime=1731527059577; duration=0sec 2024-11-13T19:44:20,050 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:20,050 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:20,053 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:20,054 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0e3223ab95c64f2b957ed3fe25680e12 is 50, key is test_row_0/C:col10/1731527058990/Put/seqid=0 2024-11-13T19:44:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742127_1303 (size=12915) 2024-11-13T19:44:20,091 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0e3223ab95c64f2b957ed3fe25680e12 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0e3223ab95c64f2b957ed3fe25680e12 2024-11-13T19:44:20,108 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 0e3223ab95c64f2b957ed3fe25680e12(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:20,108 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,108 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=11, startTime=1731527059579; duration=0sec 2024-11-13T19:44:20,108 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:20,108 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:20,155 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/b56ee06a11b94b8591c850a609ae54c6 2024-11-13T19:44:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c0e051c4203c488e9fbce77f72635f03 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03 2024-11-13T19:44:20,176 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T19:44:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c8be12670010417bbd49aa97f5cee8a9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9 2024-11-13T19:44:20,185 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T19:44:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527120184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527120184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/b56ee06a11b94b8591c850a609ae54c6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527120184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527120184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:20,195 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T19:44:20,197 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=46.96 KB/48090 for 6f4ececb48a6d1da14a168d390e345dd in 523ms, sequenceid=558, compaction requested=false 2024-11-13T19:44:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T19:44:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T19:44:20,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T19:44:20,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1450 sec 2024-11-13T19:44:20,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1540 sec 2024-11-13T19:44:20,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:20,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/a1297d68b4694cd2a2e61907ca561168 is 50, key is test_row_0/A:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742128_1304 (size=12301) 2024-11-13T19:44:20,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/a1297d68b4694cd2a2e61907ca561168 2024-11-13T19:44:20,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ede025220aec4d958adbefef2c5686e1 is 50, key is test_row_0/B:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742129_1305 (size=12301) 2024-11-13T19:44:20,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ede025220aec4d958adbefef2c5686e1 2024-11-13T19:44:20,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/1d794a888d9c49f3bb31bb110ede8766 is 50, key is test_row_0/C:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742130_1306 (size=12301) 2024-11-13T19:44:20,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527060451, max timestamp=1731527060483Num uniques 859; 2024-11-13T19:44:20,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:20,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527060451, max timestamp=1731527060483Num uniques 859; 2024-11-13T19:44:20,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:20,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527060451, max timestamp=1731527060483Num uniques 859; 2024-11-13T19:44:20,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527120515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527120515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527120522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527120522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527120522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527120634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527120634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527120634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527120637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527120643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527120843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527120843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527120843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527120854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527120863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:20,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/1d794a888d9c49f3bb31bb110ede8766 2024-11-13T19:44:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/a1297d68b4694cd2a2e61907ca561168 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168 2024-11-13T19:44:20,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168, entries=150, sequenceid=572, filesize=12.0 K 2024-11-13T19:44:20,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ede025220aec4d958adbefef2c5686e1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1 2024-11-13T19:44:20,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1, entries=150, sequenceid=572, filesize=12.0 K 2024-11-13T19:44:20,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/1d794a888d9c49f3bb31bb110ede8766 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766 2024-11-13T19:44:20,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766, entries=150, sequenceid=572, filesize=12.0 K 2024-11-13T19:44:20,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6f4ececb48a6d1da14a168d390e345dd in 512ms, sequenceid=572, compaction requested=true 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:20,891 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:44:20,893 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:20,893 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:20,893 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:20,893 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:20,893 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/2cfe874205e647fe8f8862fb8126bb16, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=36.6 K 2024-11-13T19:44:20,894 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cfe874205e647fe8f8862fb8126bb16, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527056803 2024-11-13T19:44:20,900 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e051c4203c488e9fbce77f72635f03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731527059196 2024-11-13T19:44:20,904 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:20,904 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1297d68b4694cd2a2e61907ca561168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:20,904 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:20,904 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:20,904 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0f8c4d6655374bb8a81b397b07e13bb0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=36.6 K 2024-11-13T19:44:20,907 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8c4d6655374bb8a81b397b07e13bb0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527056803 2024-11-13T19:44:20,913 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c8be12670010417bbd49aa97f5cee8a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731527059196 2024-11-13T19:44:20,914 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ede025220aec4d958adbefef2c5686e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:20,925 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:20,925 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/eb8777d0fa3249c09b9616e992e39520 is 50, key is test_row_0/A:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,929 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:20,929 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/d8404a44cfad4cf7922bdf24999cfd5f is 50, key is test_row_0/B:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742131_1307 (size=13017) 2024-11-13T19:44:20,949 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/eb8777d0fa3249c09b9616e992e39520 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/eb8777d0fa3249c09b9616e992e39520 2024-11-13T19:44:20,956 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into eb8777d0fa3249c09b9616e992e39520(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:20,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,956 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527060891; duration=0sec 2024-11-13T19:44:20,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:20,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:20,957 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:20,958 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:20,958 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:20,958 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:20,958 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0e3223ab95c64f2b957ed3fe25680e12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=36.6 K 2024-11-13T19:44:20,959 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3223ab95c64f2b957ed3fe25680e12, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731527056803 2024-11-13T19:44:20,959 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b56ee06a11b94b8591c850a609ae54c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1731527059196 2024-11-13T19:44:20,959 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d794a888d9c49f3bb31bb110ede8766, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:20,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742132_1308 (size=13017) 2024-11-13T19:44:20,976 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:20,976 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/1b43b120c05f401e8999fb057bee2363 is 50, key is test_row_0/C:col10/1731527059838/Put/seqid=0 2024-11-13T19:44:20,984 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/d8404a44cfad4cf7922bdf24999cfd5f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d8404a44cfad4cf7922bdf24999cfd5f 2024-11-13T19:44:20,991 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into d8404a44cfad4cf7922bdf24999cfd5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:20,993 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:20,993 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527060891; duration=0sec 2024-11-13T19:44:20,993 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:20,993 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:20,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742133_1309 (size=13017) 2024-11-13T19:44:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T19:44:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:21,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:21,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/4e288b9b0ba7412d8f270e84d24f649c is 50, key is test_row_0/A:col10/1731527060513/Put/seqid=0 2024-11-13T19:44:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527121167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527121168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742134_1310 (size=12301) 2024-11-13T19:44:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527121170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527121171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527121174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:44:21,204 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:21,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T19:44:21,211 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:21,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:21,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527121284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527121284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527121284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527121284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:21,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,401 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/1b43b120c05f401e8999fb057bee2363 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1b43b120c05f401e8999fb057bee2363 2024-11-13T19:44:21,406 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 1b43b120c05f401e8999fb057bee2363(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:21,406 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:21,406 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=13, startTime=1731527060891; duration=0sec 2024-11-13T19:44:21,406 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:21,406 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:21,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527121493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527121494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527121494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527121494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:21,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/4e288b9b0ba7412d8f270e84d24f649c 2024-11-13T19:44:21,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/0b775207fee14770b3a193d6af00479d is 50, key is test_row_0/B:col10/1731527060513/Put/seqid=0 2024-11-13T19:44:21,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742135_1311 (size=12301) 2024-11-13T19:44:21,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527121683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527121804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527121804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527121804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527121804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:21,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:21,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:21,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:22,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/0b775207fee14770b3a193d6af00479d 2024-11-13T19:44:22,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cfe30ada614b487490eee15ef751b06d is 50, key is test_row_0/C:col10/1731527060513/Put/seqid=0 2024-11-13T19:44:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742136_1312 (size=12301) 2024-11-13T19:44:22,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cfe30ada614b487490eee15ef751b06d 2024-11-13T19:44:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/4e288b9b0ba7412d8f270e84d24f649c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c 2024-11-13T19:44:22,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c, entries=150, sequenceid=612, filesize=12.0 K 2024-11-13T19:44:22,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/0b775207fee14770b3a193d6af00479d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d 2024-11-13T19:44:22,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d, entries=150, sequenceid=612, filesize=12.0 K 2024-11-13T19:44:22,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/cfe30ada614b487490eee15ef751b06d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d 2024-11-13T19:44:22,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d, entries=150, sequenceid=612, filesize=12.0 K 2024-11-13T19:44:22,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 6f4ececb48a6d1da14a168d390e345dd in 947ms, sequenceid=612, compaction requested=false 2024-11-13T19:44:22,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:22,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:44:22,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:22,141 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/85dbf17abfd248cd9b2adbd4c20cbaab is 50, key is test_row_0/A:col10/1731527061166/Put/seqid=0 2024-11-13T19:44:22,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742137_1313 (size=12301) 2024-11-13T19:44:22,171 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/85dbf17abfd248cd9b2adbd4c20cbaab 2024-11-13T19:44:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/6cc5a11c1a144c11b11ccc5f1afa9377 is 50, key is test_row_0/B:col10/1731527061166/Put/seqid=0 2024-11-13T19:44:22,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742138_1314 (size=12301) 2024-11-13T19:44:22,195 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/6cc5a11c1a144c11b11ccc5f1afa9377 2024-11-13T19:44:22,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9139b7025c3b4cbebf5ceb810a4b23d7 is 50, key is test_row_0/C:col10/1731527061166/Put/seqid=0 2024-11-13T19:44:22,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742139_1315 (size=12301) 2024-11-13T19:44:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:22,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:22,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527062340, max timestamp=1731527062352Num uniques 859; 2024-11-13T19:44:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527062340, max timestamp=1731527062352Num uniques 859; 2024-11-13T19:44:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527062340, max timestamp=1731527062352Num uniques 859; 2024-11-13T19:44:22,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527122395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527122395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527122397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527062382, max timestamp=1731527062402Num uniques 1018; 2024-11-13T19:44:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527122404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T19:44:22,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527062382, max timestamp=1731527062402Num uniques 1018; 2024-11-13T19:44:22,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-13T19:44:22,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527062382, max timestamp=1731527062402Num uniques 1018; 2024-11-13T19:44:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T19:44:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731527062540, max timestamp=1731527062554Num uniques 1027; 2024-11-13T19:44:22,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-13T19:44:22,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731527062540, max timestamp=1731527062554Num uniques 1027; 2024-11-13T19:44:22,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-13T19:44:22,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527062543, max timestamp=1731527062554Num uniques 1009; 2024-11-13T19:44:22,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-13T19:44:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527122591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527122596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527122598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527122599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,638 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9139b7025c3b4cbebf5ceb810a4b23d7 2024-11-13T19:44:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/85dbf17abfd248cd9b2adbd4c20cbaab as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab 2024-11-13T19:44:22,651 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab, entries=150, sequenceid=622, filesize=12.0 K 2024-11-13T19:44:22,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/6cc5a11c1a144c11b11ccc5f1afa9377 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377 2024-11-13T19:44:22,658 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377, entries=150, sequenceid=622, filesize=12.0 K 2024-11-13T19:44:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9139b7025c3b4cbebf5ceb810a4b23d7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7 2024-11-13T19:44:22,665 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7, entries=150, sequenceid=622, filesize=12.0 K 2024-11-13T19:44:22,666 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=143.13 KB/146560 for 6f4ececb48a6d1da14a168d390e345dd in 524ms, sequenceid=622, compaction requested=true 2024-11-13T19:44:22,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:22,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:22,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T19:44:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T19:44:22,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T19:44:22,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-13T19:44:22,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4660 sec 2024-11-13T19:44:22,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527062581, max timestamp=1731527062696Num uniques 1136; 2024-11-13T19:44:22,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-13T19:44:22,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527062581, max timestamp=1731527062696Num uniques 1036; 2024-11-13T19:44:22,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:44:22,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527062581, max timestamp=1731527062696Num uniques 1136; 2024-11-13T19:44:22,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-13T19:44:22,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=69.33 KB heapSize=187.27 KB 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:22,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/905bfa1af15240818082ce010611bebc is 50, key is test_row_0/A:col10/1731527062589/Put/seqid=0 2024-11-13T19:44:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742140_1316 (size=14741) 2024-11-13T19:44:22,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527062723, max timestamp=1731527062740Num uniques 859; 2024-11-13T19:44:22,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:44:22,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527062723, max timestamp=1731527062740Num uniques 859; 2024-11-13T19:44:22,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:44:22,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527062723, max timestamp=1731527062740Num uniques 859; 2024-11-13T19:44:22,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:44:22,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731527062748, max timestamp=1731527062774Num uniques 1168; 2024-11-13T19:44:22,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-13T19:44:22,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731527062748, max timestamp=1731527062774Num uniques 1168; 2024-11-13T19:44:22,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-13T19:44:22,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527062758, max timestamp=1731527062774Num uniques 1168; 2024-11-13T19:44:22,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=6870, suffix heap size=149440,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=150 2024-11-13T19:44:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527122809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527122809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:22,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731527062804, max timestamp=1731527062813Num uniques 1177; 2024-11-13T19:44:22,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-13T19:44:22,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731527062804, max timestamp=1731527062813Num uniques 1177; 2024-11-13T19:44:22,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-13T19:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:22,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527062806, max timestamp=1731527062813Num uniques 1009; 2024-11-13T19:44:22,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-13T19:44:22,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731527062840, max timestamp=1731527062860Num uniques 1136; 2024-11-13T19:44:22,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-13T19:44:22,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731527062840, max timestamp=1731527062860Num uniques 1136; 2024-11-13T19:44:22,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-13T19:44:22,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731527062840, max timestamp=1731527062860Num uniques 1036; 2024-11-13T19:44:22,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T19:44:22,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731527062929, max timestamp=1731527062972Num uniques 1095; 2024-11-13T19:44:22,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-13T19:44:22,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527062970, max timestamp=1731527062972Num uniques 1095; 2024-11-13T19:44:22,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527062970, max timestamp=1731527062972Num uniques 1095; 2024-11-13T19:44:22,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T19:44:22,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T19:44:23,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731527063045, max timestamp=1731527063070Num uniques 1254; 2024-11-13T19:44:23,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731527063045, max timestamp=1731527063070Num uniques 1054; 2024-11-13T19:44:23,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48264, new segment data size=9334, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-13T19:44:23,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731527063045, max timestamp=1731527063070Num uniques 1054; 2024-11-13T19:44:23,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48264, new segment data size=9334, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-13T19:44:23,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=57424, new segment data size=9334, suffix heap size=158384,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=204 2024-11-13T19:44:23,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527063107, max timestamp=1731527063119Num uniques 1063; 2024-11-13T19:44:23,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-13T19:44:23,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527063107, max timestamp=1731527063119Num uniques 1063; 2024-11-13T19:44:23,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-13T19:44:23,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527063107, max timestamp=1731527063119Num uniques 1063; 2024-11-13T19:44:23,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-13T19:44:23,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/905bfa1af15240818082ce010611bebc 2024-11-13T19:44:23,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7d3a127405c44503966425b16ad30ff4 is 50, key is test_row_0/B:col10/1731527062589/Put/seqid=0 2024-11-13T19:44:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742141_1317 (size=12301) 2024-11-13T19:44:23,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731527063145, max timestamp=1731527063158Num uniques 1072; 2024-11-13T19:44:23,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-13T19:44:23,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731527063145, max timestamp=1731527063158Num uniques 1072; 2024-11-13T19:44:23,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-13T19:44:23,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731527063145, max timestamp=1731527063158Num uniques 1072; 2024-11-13T19:44:23,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-13T19:44:23,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527123192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:23,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731527063184, max timestamp=1731527063197Num uniques 1131; 2024-11-13T19:44:23,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-13T19:44:23,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527063185, max timestamp=1731527063198Num uniques 1131; 2024-11-13T19:44:23,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51786, new segment data size=6870, suffix heap size=145592,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=150 2024-11-13T19:44:23,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:23,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:23,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:23,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10566, totalHeapSize=24264, min timestamp=1731527063184, max timestamp=1731527063198Num uniques 1131; 2024-11-13T19:44:23,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51786, new segment data size=10566, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-13T19:44:23,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731527063217, max timestamp=1731527063241Num uniques 1090; 2024-11-13T19:44:23,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-13T19:44:23,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731527063217, max timestamp=1731527063241Num uniques 1009; 2024-11-13T19:44:23,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=8700, suffix heap size=132904,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=190 2024-11-13T19:44:23,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731527063217, max timestamp=1731527063241Num uniques 1090; 2024-11-13T19:44:23,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731527063256, max timestamp=1731527063278Num uniques 1049; 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731527063256, max timestamp=1731527063278Num uniques 1049; 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731527063256, max timestamp=1731527063278Num uniques 1049; 2024-11-13T19:44:23,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-13T19:44:23,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:23,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731527063307, max timestamp=1731527063321Num uniques 1208; 2024-11-13T19:44:23,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=9520, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-11-13T19:44:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527123317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:23,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731527063307, max timestamp=1731527063321Num uniques 1208; 2024-11-13T19:44:23,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=9520, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-11-13T19:44:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:23,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527063307, max timestamp=1731527063321Num uniques 1208; 2024-11-13T19:44:23,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=6870, suffix heap size=153600,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=150 2024-11-13T19:44:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:44:23,374 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:23,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T19:44:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:23,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:23,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:23,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527063377, max timestamp=1731527063407Num uniques 1067; 2024-11-13T19:44:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-13T19:44:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527063377, max timestamp=1731527063407Num uniques 1009; 2024-11-13T19:44:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-11-13T19:44:23,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731527063377, max timestamp=1731527063407Num uniques 1067; 2024-11-13T19:44:23,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-13T19:44:23,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731527063448, max timestamp=1731527063456Num uniques 1026; 2024-11-13T19:44:23,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-13T19:44:23,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731527063448, max timestamp=1731527063456Num uniques 1026; 2024-11-13T19:44:23,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-13T19:44:23,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731527063448, max timestamp=1731527063456Num uniques 1026; 2024-11-13T19:44:23,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-13T19:44:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:23,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731527063478, max timestamp=1731527063493Num uniques 1035; 2024-11-13T19:44:23,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-13T19:44:23,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731527063478, max timestamp=1731527063493Num uniques 1035; 2024-11-13T19:44:23,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-13T19:44:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731527063478, max timestamp=1731527063493Num uniques 1035; 2024-11-13T19:44:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-13T19:44:23,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731527063518, max timestamp=1731527063530Num uniques 1094; 2024-11-13T19:44:23,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-13T19:44:23,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731527063518, max timestamp=1731527063530Num uniques 1094; 2024-11-13T19:44:23,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-13T19:44:23,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731527063518, max timestamp=1731527063530Num uniques 1094; 2024-11-13T19:44:23,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-13T19:44:23,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7d3a127405c44503966425b16ad30ff4 2024-11-13T19:44:23,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0477cbf2e3ce43bf97d8c798daf953a3 is 50, key is test_row_0/C:col10/1731527062589/Put/seqid=0 2024-11-13T19:44:23,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=303, cellSize=13875, totalHeapSize=31752, min timestamp=1731527063543, max timestamp=1731527063570Num uniques 1053; 2024-11-13T19:44:23,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=13875, suffix heap size=137480,new segment heap size=31752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=303 2024-11-13T19:44:23,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742142_1318 (size=12301) 2024-11-13T19:44:23,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9288, totalHeapSize=21352, min timestamp=1731527063543, max timestamp=1731527063570Num uniques 1053; 2024-11-13T19:44:23,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48218, new segment data size=9288, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-13T19:44:23,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731527063543, max timestamp=1731527063570Num uniques 1053; 2024-11-13T19:44:23,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-13T19:44:23,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731527063586, max timestamp=1731527063610Num uniques 1062; 2024-11-13T19:44:23,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=11992, suffix heap size=138416,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=262 2024-11-13T19:44:23,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731527063586, max timestamp=1731527063610Num uniques 1062; 2024-11-13T19:44:23,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731527063586, max timestamp=1731527063610Num uniques 1162; 2024-11-13T19:44:23,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53212, new segment data size=11992, suffix heap size=148816,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1162, new segment cells count=262 2024-11-13T19:44:23,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=11992, suffix heap size=138416,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=262 2024-11-13T19:44:23,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1731527063628, max timestamp=1731527063650Num uniques 1121; 2024-11-13T19:44:23,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1731527063628, max timestamp=1731527063650Num uniques 1121; 2024-11-13T19:44:23,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=12406, suffix heap size=144552,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=271 2024-11-13T19:44:23,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=12406, suffix heap size=144552,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=271 2024-11-13T19:44:23,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1731527063628, max timestamp=1731527063650Num uniques 1121; 2024-11-13T19:44:23,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=12406, suffix heap size=144552,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=271 2024-11-13T19:44:23,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:23,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8230, totalHeapSize=18960, min timestamp=1731527063687, max timestamp=1731527063732Num uniques 1130; 2024-11-13T19:44:23,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51740, new segment data size=8230, suffix heap size=145488,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=180 2024-11-13T19:44:23,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8235, totalHeapSize=18960, min timestamp=1731527063687, max timestamp=1731527063732Num uniques 1130; 2024-11-13T19:44:23,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51745, new segment data size=8235, suffix heap size=145488,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=180 2024-11-13T19:44:23,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8232, totalHeapSize=18960, min timestamp=1731527063687, max timestamp=1731527063732Num uniques 1130; 2024-11-13T19:44:23,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51742, new segment data size=8232, suffix heap size=145488,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=180 2024-11-13T19:44:23,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=389, cellSize=17814, totalHeapSize=40696, min timestamp=1731527063761, max timestamp=1731527063781Num uniques 1039; 2024-11-13T19:44:23,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=17814, suffix heap size=136024,new segment heap size=40696 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=389 2024-11-13T19:44:23,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=389, cellSize=17814, totalHeapSize=40696, min timestamp=1731527063761, max timestamp=1731527063781Num uniques 1039; 2024-11-13T19:44:23,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=17814, suffix heap size=136024,new segment heap size=40696 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=389 2024-11-13T19:44:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731527063774, max timestamp=1731527063781Num uniques 1039; 2024-11-13T19:44:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-11-13T19:44:23,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731527063808, max timestamp=1731527063820Num uniques 1248; 2024-11-13T19:44:23,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57158, new segment data size=13648, suffix heap size=157760,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1248, new segment cells count=298 2024-11-13T19:44:23,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731527063808, max timestamp=1731527063820Num uniques 1098; 2024-11-13T19:44:23,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=13648, suffix heap size=142160,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=298 2024-11-13T19:44:23,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731527063808, max timestamp=1731527063820Num uniques 1248; 2024-11-13T19:44:23,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=57158, new segment data size=13648, suffix heap size=157760,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1248, new segment cells count=298 2024-11-13T19:44:23,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:23,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:23,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:23,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731527063861, max timestamp=1731527063868Num uniques 1157; 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731527063861, max timestamp=1731527063868Num uniques 1157; 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731527063861, max timestamp=1731527063868Num uniques 1157; 2024-11-13T19:44:23,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-13T19:44:23,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731527063910, max timestamp=1731527063937Num uniques 1066; 2024-11-13T19:44:23,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731527063910, max timestamp=1731527063937Num uniques 1066; 2024-11-13T19:44:23,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-13T19:44:23,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-13T19:44:23,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731527063910, max timestamp=1731527063933Num uniques 1066; 2024-11-13T19:44:23,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-13T19:44:23,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0477cbf2e3ce43bf97d8c798daf953a3 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731527063970, max timestamp=1731527063984Num uniques 1025; 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1731527063966, max timestamp=1731527063984Num uniques 1025; 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=14880, suffix heap size=134568,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=325 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731527063970, max timestamp=1731527063984Num uniques 1025; 2024-11-13T19:44:23,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-13T19:44:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/905bfa1af15240818082ce010611bebc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc 2024-11-13T19:44:23,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc, entries=200, sequenceid=694, filesize=14.4 K 2024-11-13T19:44:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7d3a127405c44503966425b16ad30ff4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4 2024-11-13T19:44:24,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4, entries=150, sequenceid=694, filesize=12.0 K 2024-11-13T19:44:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0477cbf2e3ce43bf97d8c798daf953a3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3 2024-11-13T19:44:24,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3, entries=150, sequenceid=694, filesize=12.0 K 2024-11-13T19:44:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:24,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~63.66 KB/65184, currentSize=46.96 KB/48090 for 6f4ececb48a6d1da14a168d390e345dd in 1328ms, sequenceid=694, compaction requested=true 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:24,030 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:24,030 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:24,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=60.38 KB heapSize=159.66 KB 2024-11-13T19:44:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,032 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:24,033 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:24,033 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,033 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d8404a44cfad4cf7922bdf24999cfd5f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=48.8 K 2024-11-13T19:44:24,033 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:24,033 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:24,033 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,033 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/eb8777d0fa3249c09b9616e992e39520, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=51.1 K 2024-11-13T19:44:24,034 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb8777d0fa3249c09b9616e992e39520, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:24,034 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d8404a44cfad4cf7922bdf24999cfd5f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:24,035 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e288b9b0ba7412d8f270e84d24f649c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731527060513 2024-11-13T19:44:24,035 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b775207fee14770b3a193d6af00479d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731527060513 2024-11-13T19:44:24,036 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc5a11c1a144c11b11ccc5f1afa9377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1731527061165 2024-11-13T19:44:24,036 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85dbf17abfd248cd9b2adbd4c20cbaab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1731527061165 2024-11-13T19:44:24,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c281351b92ff4a4380874bfb638e970d is 50, key is test_row_0/A:col10/1731527063982/Put/seqid=0 2024-11-13T19:44:24,040 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905bfa1af15240818082ce010611bebc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731527062581 2024-11-13T19:44:24,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3a127405c44503966425b16ad30ff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731527062581 2024-11-13T19:44:24,052 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:24,053 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/b4df22cb219145059825e552dbe50012 is 50, key is test_row_0/A:col10/1731527062589/Put/seqid=0 2024-11-13T19:44:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742143_1319 (size=12301) 2024-11-13T19:44:24,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c281351b92ff4a4380874bfb638e970d 2024-11-13T19:44:24,060 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#275 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:24,061 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/d5ff896de9f34e5b87c3261114f34d68 is 50, key is test_row_0/B:col10/1731527062589/Put/seqid=0 2024-11-13T19:44:24,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/2c82c90b9aa148c3a7b9e4462ee19627 is 50, key is test_row_0/B:col10/1731527063982/Put/seqid=0 2024-11-13T19:44:24,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742144_1320 (size=13153) 2024-11-13T19:44:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527124090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742145_1321 (size=13153) 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527124092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527124092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527124092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742146_1322 (size=12301) 2024-11-13T19:44:24,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/2c82c90b9aa148c3a7b9e4462ee19627 2024-11-13T19:44:24,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/991fdb53542345a397a00c26a699a123 is 50, key is test_row_0/C:col10/1731527063982/Put/seqid=0 2024-11-13T19:44:24,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742147_1323 (size=12301) 2024-11-13T19:44:24,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1132 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/991fdb53542345a397a00c26a699a123 2024-11-13T19:44:24,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/c281351b92ff4a4380874bfb638e970d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d 2024-11-13T19:44:24,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d, entries=150, sequenceid=1132, filesize=12.0 K 2024-11-13T19:44:24,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/2c82c90b9aa148c3a7b9e4462ee19627 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627 2024-11-13T19:44:24,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627, entries=150, sequenceid=1132, filesize=12.0 K 2024-11-13T19:44:24,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/991fdb53542345a397a00c26a699a123 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123 2024-11-13T19:44:24,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123, entries=150, sequenceid=1132, filesize=12.0 K 2024-11-13T19:44:24,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~179.53 KB/183840, currentSize=114.05 KB/116790 for 6f4ececb48a6d1da14a168d390e345dd in 184ms, sequenceid=1132, compaction requested=true 2024-11-13T19:44:24,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:24,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:24,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:44:24,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:24,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:44:24,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:24,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:44:24,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527064082, max timestamp=1731527064224Num uniques 859; 2024-11-13T19:44:24,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:24,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527064082, max timestamp=1731527064224Num uniques 859; 2024-11-13T19:44:24,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:24,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527064082, max timestamp=1731527064224Num uniques 859; 2024-11-13T19:44:24,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:24,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-13T19:44:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:24,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9d8d06dd419c4532a13926186cdd6760 is 50, key is test_row_0/A:col10/1731527064222/Put/seqid=0 2024-11-13T19:44:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742148_1324 (size=12301) 2024-11-13T19:44:24,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527064278, max timestamp=1731527064317Num uniques 859; 2024-11-13T19:44:24,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:44:24,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527064278, max timestamp=1731527064317Num uniques 859; 2024-11-13T19:44:24,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527064278, max timestamp=1731527064317Num uniques 859; 2024-11-13T19:44:24,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:44:24,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T19:44:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 373 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527124379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527124382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527124382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527064357, max timestamp=1731527064388Num uniques 1009; 2024-11-13T19:44:24,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-13T19:44:24,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527064357, max timestamp=1731527064388Num uniques 1009; 2024-11-13T19:44:24,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-13T19:44:24,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527064357, max timestamp=1731527064388Num uniques 1009; 2024-11-13T19:44:24,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-13T19:44:24,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,511 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/b4df22cb219145059825e552dbe50012 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/b4df22cb219145059825e552dbe50012 2024-11-13T19:44:24,514 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/d5ff896de9f34e5b87c3261114f34d68 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d5ff896de9f34e5b87c3261114f34d68 2024-11-13T19:44:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527124495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 374 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527124504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527124504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527124506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,521 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into b4df22cb219145059825e552dbe50012(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:24,521 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:24,521 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=12, startTime=1731527064030; duration=0sec 2024-11-13T19:44:24,521 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:44:24,521 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:24,521 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:24,521 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527124514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,524 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into d5ff896de9f34e5b87c3261114f34d68(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:24,524 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:24,524 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=12, startTime=1731527064030; duration=0sec 2024-11-13T19:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,527 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:24,527 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:24,527 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,527 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1b43b120c05f401e8999fb057bee2363, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=60.8 K 2024-11-13T19:44:24,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b43b120c05f401e8999fb057bee2363, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731527059838 2024-11-13T19:44:24,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe30ada614b487490eee15ef751b06d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731527060513 2024-11-13T19:44:24,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9139b7025c3b4cbebf5ceb810a4b23d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1731527061165 2024-11-13T19:44:24,529 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0477cbf2e3ce43bf97d8c798daf953a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731527062581 2024-11-13T19:44:24,529 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991fdb53542345a397a00c26a699a123, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1132, earliestPutTs=1731527063982 2024-11-13T19:44:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. because compaction request was cancelled 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:24,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:44:24,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:44:24,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:44:24,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. because compaction request was cancelled 2024-11-13T19:44:24,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:24,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:44:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:44:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:44:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. because compaction request was cancelled 2024-11-13T19:44:24,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:24,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:24,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/4e199ad5880a451c9fab9c8449e5f0bf is 50, key is test_row_0/C:col10/1731527063982/Put/seqid=0 2024-11-13T19:44:24,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742149_1325 (size=13187) 2024-11-13T19:44:24,623 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/4e199ad5880a451c9fab9c8449e5f0bf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4e199ad5880a451c9fab9c8449e5f0bf 2024-11-13T19:44:24,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527124624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527124634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,645 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 4e199ad5880a451c9fab9c8449e5f0bf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:24,645 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:24,645 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=11, startTime=1731527064215; duration=0sec 2024-11-13T19:44:24,645 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:24,645 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:24,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9d8d06dd419c4532a13926186cdd6760 2024-11-13T19:44:24,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/8e676bde45de4e44a01c53ca89ef766a is 50, key is test_row_0/B:col10/1731527064222/Put/seqid=0 2024-11-13T19:44:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527124723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527124723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527124724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742150_1326 (size=12301) 2024-11-13T19:44:24,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527124833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527124844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:24,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:24,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527125044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527125044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527125044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:25,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527125143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527125154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/8e676bde45de4e44a01c53ca89ef766a 2024-11-13T19:44:25,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/7d737f8fd7fb490699529145975e3a7b is 50, key is test_row_0/C:col10/1731527064222/Put/seqid=0 2024-11-13T19:44:25,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742151_1327 (size=12301) 2024-11-13T19:44:25,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1156 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/7d737f8fd7fb490699529145975e3a7b 2024-11-13T19:44:25,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/9d8d06dd419c4532a13926186cdd6760 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760 2024-11-13T19:44:25,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760, entries=150, sequenceid=1156, filesize=12.0 K 2024-11-13T19:44:25,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/8e676bde45de4e44a01c53ca89ef766a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a 2024-11-13T19:44:25,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a, entries=150, sequenceid=1156, filesize=12.0 K 2024-11-13T19:44:25,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/7d737f8fd7fb490699529145975e3a7b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b 2024-11-13T19:44:25,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b, entries=150, sequenceid=1156, filesize=12.0 K 2024-11-13T19:44:25,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=136.42 KB/139690 for 6f4ececb48a6d1da14a168d390e345dd in 1019ms, sequenceid=1156, compaction requested=true 2024-11-13T19:44:25,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:25,251 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:25,252 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:25,254 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:25,254 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:25,254 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:25,254 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/b4df22cb219145059825e552dbe50012, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=36.9 K 2024-11-13T19:44:25,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:25,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:25,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:25,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:25,255 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:25,255 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:25,255 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:25,255 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d5ff896de9f34e5b87c3261114f34d68, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=36.9 K 2024-11-13T19:44:25,256 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4df22cb219145059825e552dbe50012, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731527062581 2024-11-13T19:44:25,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ff896de9f34e5b87c3261114f34d68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731527062581 2024-11-13T19:44:25,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c82c90b9aa148c3a7b9e4462ee19627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1132, earliestPutTs=1731527063982 2024-11-13T19:44:25,256 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c281351b92ff4a4380874bfb638e970d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1132, earliestPutTs=1731527063982 2024-11-13T19:44:25,257 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e676bde45de4e44a01c53ca89ef766a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1156, earliestPutTs=1731527064082 2024-11-13T19:44:25,257 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d8d06dd419c4532a13926186cdd6760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1156, earliestPutTs=1731527064082 2024-11-13T19:44:25,271 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:25,272 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/ade1a7856c9f4f4bb4ffd486ee0c001e is 50, key is test_row_0/A:col10/1731527064222/Put/seqid=0 2024-11-13T19:44:25,275 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:25,276 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/3bfcc8e224b7476b95032cfba6e520fa is 50, key is test_row_0/B:col10/1731527064222/Put/seqid=0 2024-11-13T19:44:25,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742152_1328 (size=13255) 2024-11-13T19:44:25,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:44:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:25,279 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=136.42 KB heapSize=389.22 KB 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:25,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7255bf725a7b47dd88d5762190f8d72e is 50, key is test_row_0/A:col10/1731527064488/Put/seqid=0 2024-11-13T19:44:25,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742153_1329 (size=13255) 2024-11-13T19:44:25,313 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/3bfcc8e224b7476b95032cfba6e520fa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/3bfcc8e224b7476b95032cfba6e520fa 2024-11-13T19:44:25,342 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into 3bfcc8e224b7476b95032cfba6e520fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:25,342 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:25,342 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527065252; duration=0sec 2024-11-13T19:44:25,342 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:25,342 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:25,342 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:44:25,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742154_1330 (size=12301) 2024-11-13T19:44:25,344 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7255bf725a7b47dd88d5762190f8d72e 2024-11-13T19:44:25,344 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:44:25,344 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:44:25,345 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. because compaction request was cancelled 2024-11-13T19:44:25,345 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:25,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e88c82c6430d461fbb2362696d364db8 is 50, key is test_row_0/B:col10/1731527064488/Put/seqid=0 2024-11-13T19:44:25,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742155_1331 (size=12301) 2024-11-13T19:44:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:25,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:25,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527125577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527125579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527125582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527125654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527125673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,689 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/ade1a7856c9f4f4bb4ffd486ee0c001e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/ade1a7856c9f4f4bb4ffd486ee0c001e 2024-11-13T19:44:25,702 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into ade1a7856c9f4f4bb4ffd486ee0c001e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:25,702 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:25,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,702 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527065251; duration=0sec 2024-11-13T19:44:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527125695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,702 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:25,702 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527125695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 381 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527125695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,795 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e88c82c6430d461fbb2362696d364db8 2024-11-13T19:44:25,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0a1fa589dadb456991a1eb7af606356c is 50, key is test_row_0/C:col10/1731527064488/Put/seqid=0 2024-11-13T19:44:25,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742156_1332 (size=12301) 2024-11-13T19:44:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0a1fa589dadb456991a1eb7af606356c 2024-11-13T19:44:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7255bf725a7b47dd88d5762190f8d72e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e 2024-11-13T19:44:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527125917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527125917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527125919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:25,931 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e, entries=150, sequenceid=1213, filesize=12.0 K 2024-11-13T19:44:25,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/e88c82c6430d461fbb2362696d364db8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8 2024-11-13T19:44:25,975 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8, entries=150, sequenceid=1213, filesize=12.0 K 2024-11-13T19:44:25,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/0a1fa589dadb456991a1eb7af606356c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c 2024-11-13T19:44:26,028 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c, entries=150, sequenceid=1213, filesize=12.0 K 2024-11-13T19:44:26,029 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~389.17 KB/398512, currentSize=46.96 KB/48090 for 6f4ececb48a6d1da14a168d390e345dd in 750ms, sequenceid=1213, compaction requested=true 2024-11-13T19:44:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T19:44:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T19:44:26,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T19:44:26,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-11-13T19:44:26,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.6650 sec 2024-11-13T19:44:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:26,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:26,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/6560b1fd7e8f4cecb8fc0ace5c46b82c is 50, key is test_row_0/A:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742157_1333 (size=12301) 2024-11-13T19:44:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527126277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527066262, max timestamp=1731527066280Num uniques 859; 2024-11-13T19:44:26,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:26,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527066262, max timestamp=1731527066280Num uniques 859; 2024-11-13T19:44:26,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:26,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527066262, max timestamp=1731527066280Num uniques 859; 2024-11-13T19:44:26,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 397 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527126357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527126359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527126398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527126474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 398 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527126474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 377 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527126613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1227 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/6560b1fd7e8f4cecb8fc0ace5c46b82c 2024-11-13T19:44:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527126677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,693 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527126684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527126684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 399 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527126685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,700 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:26,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/26d9e65ee02b454ea9366458fffe230e is 50, key is test_row_0/B:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742158_1334 (size=12301) 2024-11-13T19:44:26,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1227 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/26d9e65ee02b454ea9366458fffe230e 2024-11-13T19:44:26,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9dda1b29f8b34a6a8edb40483db3e308 is 50, key is test_row_0/C:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:26,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742159_1335 (size=12301) 2024-11-13T19:44:26,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1227 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9dda1b29f8b34a6a8edb40483db3e308 2024-11-13T19:44:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/6560b1fd7e8f4cecb8fc0ace5c46b82c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c 2024-11-13T19:44:26,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c, entries=150, sequenceid=1227, filesize=12.0 K 2024-11-13T19:44:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527126933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/26d9e65ee02b454ea9366458fffe230e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e 2024-11-13T19:44:26,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e, entries=150, sequenceid=1227, filesize=12.0 K 2024-11-13T19:44:26,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/9dda1b29f8b34a6a8edb40483db3e308 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308 2024-11-13T19:44:26,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308, entries=150, sequenceid=1227, filesize=12.0 K 2024-11-13T19:44:26,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 6f4ececb48a6d1da14a168d390e345dd in 763ms, sequenceid=1227, compaction requested=true 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:26,999 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:44:26,999 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:27,002 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:27,002 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:27,002 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,002 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/ade1a7856c9f4f4bb4ffd486ee0c001e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.0 K 2024-11-13T19:44:27,002 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade1a7856c9f4f4bb4ffd486ee0c001e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1156, earliestPutTs=1731527064082 2024-11-13T19:44:27,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7255bf725a7b47dd88d5762190f8d72e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1213, earliestPutTs=1731527064488 2024-11-13T19:44:27,005 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:27,006 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:27,006 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,006 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4e199ad5880a451c9fab9c8449e5f0bf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=48.9 K 2024-11-13T19:44:27,006 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6560b1fd7e8f4cecb8fc0ace5c46b82c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:27,007 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e199ad5880a451c9fab9c8449e5f0bf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1132, earliestPutTs=1731527063982 2024-11-13T19:44:27,011 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d737f8fd7fb490699529145975e3a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1156, earliestPutTs=1731527064082 2024-11-13T19:44:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:27,012 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1fa589dadb456991a1eb7af606356c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1213, earliestPutTs=1731527064488 2024-11-13T19:44:27,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T19:44:27,013 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dda1b29f8b34a6a8edb40483db3e308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:27,032 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:27,032 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/88a9a8e414014d10a9dd2db7436065cf is 50, key is test_row_0/A:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:27,037 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:27,037 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/72d81484be1e47599d2de307575b790c is 50, key is test_row_0/C:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:27,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 is 50, key is test_row_0/A:col10/1731527066349/Put/seqid=0 2024-11-13T19:44:27,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 404 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527127055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527127071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742160_1336 (size=13357) 2024-11-13T19:44:27,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742161_1337 (size=13323) 2024-11-13T19:44:27,112 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/72d81484be1e47599d2de307575b790c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/72d81484be1e47599d2de307575b790c 2024-11-13T19:44:27,120 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 72d81484be1e47599d2de307575b790c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:27,120 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:27,120 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=12, startTime=1731527066999; duration=0sec 2024-11-13T19:44:27,120 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:27,120 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:27,120 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:27,124 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:27,124 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:27,124 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,124 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/3bfcc8e224b7476b95032cfba6e520fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.0 K 2024-11-13T19:44:27,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742162_1338 (size=12301) 2024-11-13T19:44:27,126 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bfcc8e224b7476b95032cfba6e520fa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1156, earliestPutTs=1731527064082 2024-11-13T19:44:27,127 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e88c82c6430d461fbb2362696d364db8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1213, earliestPutTs=1731527064488 2024-11-13T19:44:27,127 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d9e65ee02b454ea9366458fffe230e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:27,143 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:27,144 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ed7a51ecc1354238b11ee17ce59d60ba is 50, key is test_row_0/B:col10/1731527066236/Put/seqid=0 2024-11-13T19:44:27,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742163_1339 (size=13357) 2024-11-13T19:44:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 405 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527127174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,185 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/ed7a51ecc1354238b11ee17ce59d60ba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ed7a51ecc1354238b11ee17ce59d60ba 2024-11-13T19:44:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,191 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into ed7a51ecc1354238b11ee17ce59d60ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:27,192 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527127185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,192 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527066999; duration=0sec 2024-11-13T19:44:27,192 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:27,192 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 406 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527127384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527127405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 379 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527127454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,509 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/88a9a8e414014d10a9dd2db7436065cf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/88a9a8e414014d10a9dd2db7436065cf 2024-11-13T19:44:27,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1263 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 2024-11-13T19:44:27,546 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 88a9a8e414014d10a9dd2db7436065cf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:27,546 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:27,546 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527066999; duration=0sec 2024-11-13T19:44:27,546 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:27,546 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:27,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/781ff516590e49888d460e4c258c1f5d is 50, key is test_row_0/B:col10/1731527066349/Put/seqid=0 2024-11-13T19:44:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:44:27,570 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:27,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T19:44:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:27,599 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:27,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:27,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:27,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742164_1340 (size=12301) 2024-11-13T19:44:27,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1263 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/781ff516590e49888d460e4c258c1f5d 2024-11-13T19:44:27,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/fe87a7e1a7744749b31dc78163e53cc6 is 50, key is test_row_0/C:col10/1731527066349/Put/seqid=0 2024-11-13T19:44:27,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742165_1341 (size=12301) 2024-11-13T19:44:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:27,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 407 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527127703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527127717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:27,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:27,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:27,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:27,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:27,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:27,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1263 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/fe87a7e1a7744749b31dc78163e53cc6 2024-11-13T19:44:28,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 2024-11-13T19:44:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6, entries=150, sequenceid=1263, filesize=12.0 K 2024-11-13T19:44:28,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/781ff516590e49888d460e4c258c1f5d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d 2024-11-13T19:44:28,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d, entries=150, sequenceid=1263, filesize=12.0 K 2024-11-13T19:44:28,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/fe87a7e1a7744749b31dc78163e53cc6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6 2024-11-13T19:44:28,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6, entries=150, sequenceid=1263, filesize=12.0 K 2024-11-13T19:44:28,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6f4ececb48a6d1da14a168d390e345dd in 1092ms, sequenceid=1263, compaction requested=false 2024-11-13T19:44:28,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:28,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:28,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/278b7f9ef64940c79c55596e0863907e is 50, key is test_row_0/A:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:28,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742166_1342 (size=14741) 2024-11-13T19:44:28,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527068266, max timestamp=1731527068285Num uniques 859; 2024-11-13T19:44:28,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:28,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 419 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527128285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527068266, max timestamp=1731527068285Num uniques 859; 2024-11-13T19:44:28,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:28,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527068266, max timestamp=1731527068285Num uniques 859; 2024-11-13T19:44:28,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:28,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527128396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 421 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527128399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54968 deadline: 1731527128474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,478 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527128514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 422 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527128514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1277 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/278b7f9ef64940c79c55596e0863907e 2024-11-13T19:44:28,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/065ffd6807ba4b25810e4da451eac0cc is 50, key is test_row_0/B:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742167_1343 (size=12301) 2024-11-13T19:44:28,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54928 deadline: 1731527128714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,721 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731527128723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527128723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 423 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527128723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,728 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in 6f4ececb48a6d1da14a168d390e345dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:28,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:28,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:28,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:28,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 424 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54894 deadline: 1731527129035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:29,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54958 deadline: 1731527129037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:44:29,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1277 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/065ffd6807ba4b25810e4da451eac0cc 2024-11-13T19:44:29,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/305820b106d942498e17eba68be47308 is 50, key is test_row_0/C:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:29,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742168_1344 (size=12301) 2024-11-13T19:44:29,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1277 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/305820b106d942498e17eba68be47308 2024-11-13T19:44:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/278b7f9ef64940c79c55596e0863907e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e 2024-11-13T19:44:29,510 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-13T19:44:29,510 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,510 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,511 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-13T19:44:29,511 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,511 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e, entries=200, sequenceid=1277, filesize=14.4 K 2024-11-13T19:44:29,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/065ffd6807ba4b25810e4da451eac0cc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc 2024-11-13T19:44:29,513 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,513 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-13T19:44:29,513 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,513 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,513 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,514 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-13T19:44:29,514 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,514 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,514 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,515 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-13T19:44:29,515 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,515 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,515 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,516 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc, entries=150, sequenceid=1277, filesize=12.0 K 2024-11-13T19:44:29,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/305820b106d942498e17eba68be47308 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308 2024-11-13T19:44:29,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308, entries=150, sequenceid=1277, filesize=12.0 K 2024-11-13T19:44:29,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 6f4ececb48a6d1da14a168d390e345dd in 1313ms, sequenceid=1277, compaction requested=true 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:29,530 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f4ececb48a6d1da14a168d390e345dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:29,530 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:29,531 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:29,531 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/A is initiating minor compaction (all files) 2024-11-13T19:44:29,531 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/A in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,531 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/88a9a8e414014d10a9dd2db7436065cf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=39.5 K 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/B is initiating minor compaction (all files) 2024-11-13T19:44:29,532 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/B in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a9a8e414014d10a9dd2db7436065cf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:29,532 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ed7a51ecc1354238b11ee17ce59d60ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.1 K 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7b2efdaa6f4d48bbae18a5bcbd44c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1263, earliestPutTs=1731527066336 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7a51ecc1354238b11ee17ce59d60ba, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278b7f9ef64940c79c55596e0863907e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1277, earliestPutTs=1731527067022 2024-11-13T19:44:29,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 781ff516590e49888d460e4c258c1f5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1263, earliestPutTs=1731527066336 2024-11-13T19:44:29,533 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 065ffd6807ba4b25810e4da451eac0cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1277, earliestPutTs=1731527067022 2024-11-13T19:44:29,545 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#A#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:29,545 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:29,545 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7cc7fd5fa5fb40f4918e1943f5373746 is 50, key is test_row_0/A:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:29,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-13T19:44:29,545 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c736f5a2d2c3480a890861541ebfd669 is 50, key is test_row_0/B:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:29,546 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:29,546 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:29,546 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:29,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:29,546 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742170_1346 (size=13459) 2024-11-13T19:44:29,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7ede5e1fa93f449da2359f8afff4bec2 is 50, key is test_row_0/A:col10/1731527068396/Put/seqid=0 2024-11-13T19:44:29,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742169_1345 (size=13459) 2024-11-13T19:44:29,555 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-13T19:44:29,555 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:29,555 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,555 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:29,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:29,556 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/c736f5a2d2c3480a890861541ebfd669 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c736f5a2d2c3480a890861541ebfd669 2024-11-13T19:44:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742171_1347 (size=12301) 2024-11-13T19:44:29,561 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/B of 6f4ececb48a6d1da14a168d390e345dd into c736f5a2d2c3480a890861541ebfd669(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:29,561 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:29,561 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/B, priority=13, startTime=1731527069530; duration=0sec 2024-11-13T19:44:29,561 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:29,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:B 2024-11-13T19:44:29,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:29,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:29,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 6f4ececb48a6d1da14a168d390e345dd/C is initiating minor compaction (all files) 2024-11-13T19:44:29,563 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f4ececb48a6d1da14a168d390e345dd/C in TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,563 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/72d81484be1e47599d2de307575b790c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp, totalSize=37.0 K 2024-11-13T19:44:29,563 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d81484be1e47599d2de307575b790c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1227, earliestPutTs=1731527065576 2024-11-13T19:44:29,563 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting fe87a7e1a7744749b31dc78163e53cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1263, earliestPutTs=1731527066336 2024-11-13T19:44:29,563 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 305820b106d942498e17eba68be47308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1277, earliestPutTs=1731527067022 2024-11-13T19:44:29,570 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f4ececb48a6d1da14a168d390e345dd#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:29,570 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/28631f274e79416f87adcf10e940e0c6 is 50, key is test_row_0/C:col10/1731527068215/Put/seqid=0 2024-11-13T19:44:29,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742172_1348 (size=13425) 2024-11-13T19:44:29,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:29,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:29,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:29,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:29,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:29,957 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7cc7fd5fa5fb40f4918e1943f5373746 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7cc7fd5fa5fb40f4918e1943f5373746 2024-11-13T19:44:29,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1314 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7ede5e1fa93f449da2359f8afff4bec2 2024-11-13T19:44:29,964 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/A of 6f4ececb48a6d1da14a168d390e345dd into 7cc7fd5fa5fb40f4918e1943f5373746(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:29,964 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:29,965 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/A, priority=13, startTime=1731527069530; duration=0sec 2024-11-13T19:44:29,965 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:29,965 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:A 2024-11-13T19:44:29,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/403262f2e6cf45e3bcca21aa3ff1f549 is 50, key is test_row_0/B:col10/1731527068396/Put/seqid=0 2024-11-13T19:44:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742173_1349 (size=12301) 2024-11-13T19:44:29,978 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/28631f274e79416f87adcf10e940e0c6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/28631f274e79416f87adcf10e940e0c6 2024-11-13T19:44:29,983 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f4ececb48a6d1da14a168d390e345dd/C of 6f4ececb48a6d1da14a168d390e345dd into 28631f274e79416f87adcf10e940e0c6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:29,983 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:29,983 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., storeName=6f4ececb48a6d1da14a168d390e345dd/C, priority=13, startTime=1731527069530; duration=0sec 2024-11-13T19:44:29,983 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:29,983 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f4ececb48a6d1da14a168d390e345dd:C 2024-11-13T19:44:30,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1314 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/403262f2e6cf45e3bcca21aa3ff1f549 2024-11-13T19:44:30,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/021f91958e8f49ceb032c7bdbf74080d is 50, key is test_row_0/C:col10/1731527068396/Put/seqid=0 2024-11-13T19:44:30,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742174_1350 (size=12301) 2024-11-13T19:44:30,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1314 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/021f91958e8f49ceb032c7bdbf74080d 2024-11-13T19:44:30,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/7ede5e1fa93f449da2359f8afff4bec2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7ede5e1fa93f449da2359f8afff4bec2 2024-11-13T19:44:30,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. as already flushing 2024-11-13T19:44:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7ede5e1fa93f449da2359f8afff4bec2, entries=150, sequenceid=1314, filesize=12.0 K 2024-11-13T19:44:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:30,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/403262f2e6cf45e3bcca21aa3ff1f549 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/403262f2e6cf45e3bcca21aa3ff1f549 2024-11-13T19:44:30,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/403262f2e6cf45e3bcca21aa3ff1f549, entries=150, sequenceid=1314, filesize=12.0 K 2024-11-13T19:44:30,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/021f91958e8f49ceb032c7bdbf74080d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/021f91958e8f49ceb032c7bdbf74080d 2024-11-13T19:44:30,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/021f91958e8f49ceb032c7bdbf74080d, entries=150, sequenceid=1314, filesize=12.0 K 2024-11-13T19:44:30,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=6.71 KB/6870 for 6f4ececb48a6d1da14a168d390e345dd in 872ms, sequenceid=1314, compaction requested=false 2024-11-13T19:44:30,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:30,506 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-13T19:44:30,506 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:30,506 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:30,506 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:30,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:30,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:44:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:30,564 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1e29a479f54a4dba9b49f8b811959da2 is 50, key is test_row_0/A:col10/1731527069554/Put/seqid=0 2024-11-13T19:44:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742175_1351 (size=9857) 2024-11-13T19:44:30,973 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1322 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1e29a479f54a4dba9b49f8b811959da2 2024-11-13T19:44:30,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7dbed4809135467eb3135413c34c823b is 50, key is test_row_0/B:col10/1731527069554/Put/seqid=0 2024-11-13T19:44:30,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742176_1352 (size=9857) 2024-11-13T19:44:31,384 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1322 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7dbed4809135467eb3135413c34c823b 2024-11-13T19:44:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/f187eeca70de45188b745cd84b36737b is 50, key is test_row_0/C:col10/1731527069554/Put/seqid=0 2024-11-13T19:44:31,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742177_1353 (size=9857) 2024-11-13T19:44:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:31,797 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1322 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/f187eeca70de45188b745cd84b36737b 2024-11-13T19:44:31,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/1e29a479f54a4dba9b49f8b811959da2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1e29a479f54a4dba9b49f8b811959da2 2024-11-13T19:44:31,806 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1e29a479f54a4dba9b49f8b811959da2, entries=100, sequenceid=1322, filesize=9.6 K 2024-11-13T19:44:31,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/7dbed4809135467eb3135413c34c823b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7dbed4809135467eb3135413c34c823b 2024-11-13T19:44:31,810 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7dbed4809135467eb3135413c34c823b, entries=100, sequenceid=1322, filesize=9.6 K 2024-11-13T19:44:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/f187eeca70de45188b745cd84b36737b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/f187eeca70de45188b745cd84b36737b 2024-11-13T19:44:31,815 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/f187eeca70de45188b745cd84b36737b, entries=100, sequenceid=1322, filesize=9.6 K 2024-11-13T19:44:31,815 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6f4ececb48a6d1da14a168d390e345dd in 1252ms, sequenceid=1322, compaction requested=true 2024-11-13T19:44:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6f4ececb48a6d1da14a168d390e345dd: 2024-11-13T19:44:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T19:44:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T19:44:31,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T19:44:31,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2140 sec 2024-11-13T19:44:31,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 4.2490 sec 2024-11-13T19:44:32,746 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-13T19:44:32,746 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:32,746 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:32,746 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:32,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:32,775 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-13T19:44:32,775 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:44:32,775 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:32,775 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:32,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:44:35,814 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 325 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 360 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 174 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 182 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2156 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2051 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2055 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2132 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2075 2024-11-13T19:44:35,814 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:44:35,814 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:44:35,814 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:44:35,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:35,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:35,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:44:35,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:44:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:35,819 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527075819"}]},"ts":"1731527075819"} 2024-11-13T19:44:35,820 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:44:35,820 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:44:35,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:44:35,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, UNASSIGN}] 2024-11-13T19:44:35,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, UNASSIGN 2024-11-13T19:44:35,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=6f4ececb48a6d1da14a168d390e345dd, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:35,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, UNASSIGN because future has completed 2024-11-13T19:44:35,825 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:44:35,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:35,977 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:35,977 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:44:35,977 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 6f4ececb48a6d1da14a168d390e345dd, disabling compactions & flushes 2024-11-13T19:44:35,977 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:35,977 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:35,977 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. after waiting 0 ms 2024-11-13T19:44:35,977 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:35,977 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 6f4ececb48a6d1da14a168d390e345dd 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=A 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=B 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f4ececb48a6d1da14a168d390e345dd, store=C 2024-11-13T19:44:35,978 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:35,981 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/646e5271d48545669425657f30c2967b is 50, key is test_row_0/A:col10/1731527072744/Put/seqid=0 2024-11-13T19:44:35,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742178_1354 (size=9857) 2024-11-13T19:44:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:36,386 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1327 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/646e5271d48545669425657f30c2967b 2024-11-13T19:44:36,393 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/98c203ea3d81443ab96f624fc053fccc is 50, key is test_row_0/B:col10/1731527072744/Put/seqid=0 2024-11-13T19:44:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742179_1355 (size=9857) 2024-11-13T19:44:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:36,797 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1327 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/98c203ea3d81443ab96f624fc053fccc 2024-11-13T19:44:36,804 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c01b5a2877f94ebea485256fd31045b3 is 50, key is test_row_0/C:col10/1731527072744/Put/seqid=0 2024-11-13T19:44:36,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742180_1356 (size=9857) 2024-11-13T19:44:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:37,208 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1327 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c01b5a2877f94ebea485256fd31045b3 2024-11-13T19:44:37,219 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/A/646e5271d48545669425657f30c2967b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/646e5271d48545669425657f30c2967b 2024-11-13T19:44:37,227 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/646e5271d48545669425657f30c2967b, entries=100, sequenceid=1327, filesize=9.6 K 2024-11-13T19:44:37,228 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/B/98c203ea3d81443ab96f624fc053fccc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/98c203ea3d81443ab96f624fc053fccc 2024-11-13T19:44:37,239 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/98c203ea3d81443ab96f624fc053fccc, entries=100, sequenceid=1327, filesize=9.6 K 2024-11-13T19:44:37,240 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/.tmp/C/c01b5a2877f94ebea485256fd31045b3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c01b5a2877f94ebea485256fd31045b3 2024-11-13T19:44:37,244 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c01b5a2877f94ebea485256fd31045b3, entries=100, sequenceid=1327, filesize=9.6 K 2024-11-13T19:44:37,245 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6f4ececb48a6d1da14a168d390e345dd in 1267ms, sequenceid=1327, compaction requested=true 2024-11-13T19:44:37,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7d34ee60dab84389a08a15ff373f8fb5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/430d75f7a87c4466ab2809eeeed24f4e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c8a31d4c34a74194a148a6738c5f76f7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/54882177d1524c3dac16d5c38cae0d88, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/2cfe874205e647fe8f8862fb8126bb16, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/eb8777d0fa3249c09b9616e992e39520, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/b4df22cb219145059825e552dbe50012, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/ade1a7856c9f4f4bb4ffd486ee0c001e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/88a9a8e414014d10a9dd2db7436065cf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e] to archive 2024-11-13T19:44:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:37,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/0a83b223662d454cbf9d0ee2e999b101 2024-11-13T19:44:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1d0ffa5e81a8426597dc05dd57777fa2 2024-11-13T19:44:37,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7d34ee60dab84389a08a15ff373f8fb5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7d34ee60dab84389a08a15ff373f8fb5 2024-11-13T19:44:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/38f11ff2687542118d3b5f6d22f9848c 2024-11-13T19:44:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/d8053b73790f410d9727dd223aff68b9 2024-11-13T19:44:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/fa63ecf3cad34987a169a5daa01f8d33 2024-11-13T19:44:37,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/430d75f7a87c4466ab2809eeeed24f4e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/430d75f7a87c4466ab2809eeeed24f4e 2024-11-13T19:44:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/abfa5e3fae7046b2a88f1e1a6fd1ff9f 2024-11-13T19:44:37,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/bfe7781088b3465b891f293b9b557c1b 2024-11-13T19:44:37,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c8a31d4c34a74194a148a6738c5f76f7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c8a31d4c34a74194a148a6738c5f76f7 2024-11-13T19:44:37,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/739f128ab23b4edb8ab847d1552ca135 2024-11-13T19:44:37,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9cf743becd5b44e7b0c1a5e028d92521 2024-11-13T19:44:37,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/54882177d1524c3dac16d5c38cae0d88 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/54882177d1524c3dac16d5c38cae0d88 2024-11-13T19:44:37,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1dc68c6661434061aafa69c8c10dee89 2024-11-13T19:44:37,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/aad3a2bdf09b4cc890cc631e623cf56d 2024-11-13T19:44:37,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/56735afe4e02475b877c219e9d9f8af0 2024-11-13T19:44:37,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8a369b80a41445dc88a66d109fbd3fa4 2024-11-13T19:44:37,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/2cfe874205e647fe8f8862fb8126bb16 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/2cfe874205e647fe8f8862fb8126bb16 2024-11-13T19:44:37,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/096e10e75a53403fa7c2e9da5444f2cf 2024-11-13T19:44:37,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c0e051c4203c488e9fbce77f72635f03 2024-11-13T19:44:37,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/eb8777d0fa3249c09b9616e992e39520 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/eb8777d0fa3249c09b9616e992e39520 2024-11-13T19:44:37,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/a1297d68b4694cd2a2e61907ca561168 2024-11-13T19:44:37,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/4e288b9b0ba7412d8f270e84d24f649c 2024-11-13T19:44:37,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/85dbf17abfd248cd9b2adbd4c20cbaab 2024-11-13T19:44:37,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/905bfa1af15240818082ce010611bebc 2024-11-13T19:44:37,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/b4df22cb219145059825e552dbe50012 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/b4df22cb219145059825e552dbe50012 2024-11-13T19:44:37,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/c281351b92ff4a4380874bfb638e970d 2024-11-13T19:44:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/ade1a7856c9f4f4bb4ffd486ee0c001e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/ade1a7856c9f4f4bb4ffd486ee0c001e 2024-11-13T19:44:37,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/9d8d06dd419c4532a13926186cdd6760 2024-11-13T19:44:37,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7255bf725a7b47dd88d5762190f8d72e 2024-11-13T19:44:37,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/88a9a8e414014d10a9dd2db7436065cf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/88a9a8e414014d10a9dd2db7436065cf 2024-11-13T19:44:37,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/6560b1fd7e8f4cecb8fc0ace5c46b82c 2024-11-13T19:44:37,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/8d7b2efdaa6f4d48bbae18a5bcbd44c6 2024-11-13T19:44:37,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/278b7f9ef64940c79c55596e0863907e 2024-11-13T19:44:37,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f393f3677e4245ca932858c8f7be05ae, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/04ce271f088a4084b03c9ba95be2b68c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2589fa09e3fe43f6b702b5ce0d8e3294, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f4c3ebde605d4a1da640d04313e6aecb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0f8c4d6655374bb8a81b397b07e13bb0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d8404a44cfad4cf7922bdf24999cfd5f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d5ff896de9f34e5b87c3261114f34d68, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/3bfcc8e224b7476b95032cfba6e520fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ed7a51ecc1354238b11ee17ce59d60ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc] to archive 2024-11-13T19:44:37,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:37,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f41627da5eb347d0b576a7bdff16bd43 2024-11-13T19:44:37,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9c3e6a571a484942859accae4f0438c5 2024-11-13T19:44:37,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f393f3677e4245ca932858c8f7be05ae to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f393f3677e4245ca932858c8f7be05ae 2024-11-13T19:44:37,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/9b02e3a4b3df4ee0aa8afb4db376fc0b 2024-11-13T19:44:37,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e3132b28874a42b1beeb4c15603c5800 2024-11-13T19:44:37,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/04ce271f088a4084b03c9ba95be2b68c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/04ce271f088a4084b03c9ba95be2b68c 2024-11-13T19:44:37,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e8bbbfc01161487f964895713464489a 2024-11-13T19:44:37,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/bf02c9f53ea34c2dac74944fc9f7922e 2024-11-13T19:44:37,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2589fa09e3fe43f6b702b5ce0d8e3294 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2589fa09e3fe43f6b702b5ce0d8e3294 2024-11-13T19:44:37,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/882f33527e7746b8bac23b46768c6e5d 2024-11-13T19:44:37,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/1952393b96f54adc9c9d50637f840972 2024-11-13T19:44:37,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c5220ab666da4bf4a0b625441caa7fc1 2024-11-13T19:44:37,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f4c3ebde605d4a1da640d04313e6aecb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/f4c3ebde605d4a1da640d04313e6aecb 2024-11-13T19:44:37,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/70b32dc2d1d441ca92be9ed9a3a794c1 2024-11-13T19:44:37,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/40d1d547800f4aa59cc943371663cf80 2024-11-13T19:44:37,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ef81fd1c1e584d7fbedac49460ac3777 2024-11-13T19:44:37,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/fcc036ec4a3849928e8f585d80d60b7d 2024-11-13T19:44:37,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0f8c4d6655374bb8a81b397b07e13bb0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0f8c4d6655374bb8a81b397b07e13bb0 2024-11-13T19:44:37,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c6e3a5574e9a4f75907dc4007d2534dc 2024-11-13T19:44:37,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c8be12670010417bbd49aa97f5cee8a9 2024-11-13T19:44:37,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d8404a44cfad4cf7922bdf24999cfd5f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d8404a44cfad4cf7922bdf24999cfd5f 2024-11-13T19:44:37,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ede025220aec4d958adbefef2c5686e1 2024-11-13T19:44:37,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/0b775207fee14770b3a193d6af00479d 2024-11-13T19:44:37,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/6cc5a11c1a144c11b11ccc5f1afa9377 2024-11-13T19:44:37,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d5ff896de9f34e5b87c3261114f34d68 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/d5ff896de9f34e5b87c3261114f34d68 2024-11-13T19:44:37,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7d3a127405c44503966425b16ad30ff4 2024-11-13T19:44:37,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/2c82c90b9aa148c3a7b9e4462ee19627 2024-11-13T19:44:37,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/3bfcc8e224b7476b95032cfba6e520fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/3bfcc8e224b7476b95032cfba6e520fa 2024-11-13T19:44:37,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/8e676bde45de4e44a01c53ca89ef766a 2024-11-13T19:44:37,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/e88c82c6430d461fbb2362696d364db8 2024-11-13T19:44:37,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ed7a51ecc1354238b11ee17ce59d60ba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/ed7a51ecc1354238b11ee17ce59d60ba 2024-11-13T19:44:37,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/26d9e65ee02b454ea9366458fffe230e 2024-11-13T19:44:37,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/781ff516590e49888d460e4c258c1f5d 2024-11-13T19:44:37,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/065ffd6807ba4b25810e4da451eac0cc 2024-11-13T19:44:37,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6e1376a6c6cb449d85a339e9b487cc3c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/27cc8a94e4904c65a4637531597801fc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9e65d88a4d4d4519aaed69e336d17a31, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b1024479182c4adb9ae4d89a065cd4cc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0e3223ab95c64f2b957ed3fe25680e12, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1b43b120c05f401e8999fb057bee2363, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4e199ad5880a451c9fab9c8449e5f0bf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/72d81484be1e47599d2de307575b790c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308] to archive 2024-11-13T19:44:37,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:44:37,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/614f6b707c744588aeaaeee6fcf7066f 2024-11-13T19:44:37,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c612f240e84b4307aecf860b14afb6e0 2024-11-13T19:44:37,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6e1376a6c6cb449d85a339e9b487cc3c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6e1376a6c6cb449d85a339e9b487cc3c 2024-11-13T19:44:37,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/631efde143fa4946a4ffa11bd28f704f 2024-11-13T19:44:37,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9eb89100c2e744e9b00502a283620e43 2024-11-13T19:44:37,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/27cc8a94e4904c65a4637531597801fc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/27cc8a94e4904c65a4637531597801fc 2024-11-13T19:44:37,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/943270e5e1ce4aed9070cd46b55b3595 2024-11-13T19:44:37,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cc92ca17a4464431980d6044802e7084 2024-11-13T19:44:37,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9e65d88a4d4d4519aaed69e336d17a31 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9e65d88a4d4d4519aaed69e336d17a31 2024-11-13T19:44:37,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/58835069416941afac73754b55e8970b 2024-11-13T19:44:37,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/8d146be8db934dc087ef94e455453ab2 2024-11-13T19:44:37,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4b6772f014aa4ff2b019e4d5ad2236ac 2024-11-13T19:44:37,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b1024479182c4adb9ae4d89a065cd4cc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b1024479182c4adb9ae4d89a065cd4cc 2024-11-13T19:44:37,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/03448221d9714e21bdae2f05b89db39d 2024-11-13T19:44:37,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/54da8f98fe544d0da20fa0711cfbc3fa 2024-11-13T19:44:37,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/29f8ca2a7d36453d9af8a9c9beaad7ba 2024-11-13T19:44:37,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/06aea85c200d49bd93d5061ba532a0f9 2024-11-13T19:44:37,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0e3223ab95c64f2b957ed3fe25680e12 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0e3223ab95c64f2b957ed3fe25680e12 2024-11-13T19:44:37,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/6ba94d0d7e9944cc9511e83c8d88e0a5 2024-11-13T19:44:37,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/b56ee06a11b94b8591c850a609ae54c6 2024-11-13T19:44:37,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1b43b120c05f401e8999fb057bee2363 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1b43b120c05f401e8999fb057bee2363 2024-11-13T19:44:37,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/1d794a888d9c49f3bb31bb110ede8766 2024-11-13T19:44:37,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/cfe30ada614b487490eee15ef751b06d 2024-11-13T19:44:37,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9139b7025c3b4cbebf5ceb810a4b23d7 2024-11-13T19:44:37,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0477cbf2e3ce43bf97d8c798daf953a3 2024-11-13T19:44:37,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4e199ad5880a451c9fab9c8449e5f0bf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/4e199ad5880a451c9fab9c8449e5f0bf 2024-11-13T19:44:37,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/991fdb53542345a397a00c26a699a123 2024-11-13T19:44:37,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/7d737f8fd7fb490699529145975e3a7b 2024-11-13T19:44:37,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/0a1fa589dadb456991a1eb7af606356c 2024-11-13T19:44:37,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/72d81484be1e47599d2de307575b790c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/72d81484be1e47599d2de307575b790c 2024-11-13T19:44:37,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/9dda1b29f8b34a6a8edb40483db3e308 2024-11-13T19:44:37,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/fe87a7e1a7744749b31dc78163e53cc6 2024-11-13T19:44:37,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/305820b106d942498e17eba68be47308 2024-11-13T19:44:37,441 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/recovered.edits/1330.seqid, newMaxSeqId=1330, maxSeqId=1 2024-11-13T19:44:37,442 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. 2024-11-13T19:44:37,442 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 6f4ececb48a6d1da14a168d390e345dd: Waiting for close lock at 1731527075977Running coprocessor pre-close hooks at 1731527075977Disabling compacts and flushes for region at 1731527075977Disabling writes for close at 1731527075977Obtaining lock to block concurrent updates at 1731527075977Preparing flush snapshotting stores in 6f4ececb48a6d1da14a168d390e345dd at 1731527075977Finished memstore snapshotting TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731527075978 (+1 ms)Flushing stores of TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd. at 1731527075978Flushing 6f4ececb48a6d1da14a168d390e345dd/A: creating writer at 1731527075978Flushing 6f4ececb48a6d1da14a168d390e345dd/A: appending metadata at 1731527075981 (+3 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/A: closing flushed file at 1731527075981Flushing 6f4ececb48a6d1da14a168d390e345dd/B: creating writer at 1731527076390 (+409 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/B: appending metadata at 1731527076392 (+2 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/B: closing flushed file at 1731527076393 (+1 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/C: creating writer at 1731527076801 (+408 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/C: appending metadata at 1731527076803 (+2 ms)Flushing 6f4ececb48a6d1da14a168d390e345dd/C: closing flushed file at 1731527076803Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eaa040c: reopening flushed file at 1731527077218 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4958ef84: reopening flushed file at 1731527077227 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5efe1887: reopening flushed file at 1731527077239 (+12 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6f4ececb48a6d1da14a168d390e345dd in 1267ms, sequenceid=1327, compaction requested=true at 1731527077245 (+6 ms)Writing region close event to WAL at 1731527077435 (+190 ms)Running coprocessor post-close hooks at 1731527077442 (+7 ms)Closed at 1731527077442 2024-11-13T19:44:37,444 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:37,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=6f4ececb48a6d1da14a168d390e345dd, regionState=CLOSED 2024-11-13T19:44:37,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:37,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T19:44:37,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 6f4ececb48a6d1da14a168d390e345dd, server=54d0784d1943,36227,1731526989835 in 1.6230 sec 2024-11-13T19:44:37,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-13T19:44:37,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f4ececb48a6d1da14a168d390e345dd, UNASSIGN in 1.6280 sec 2024-11-13T19:44:37,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T19:44:37,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6350 sec 2024-11-13T19:44:37,460 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527077459"}]},"ts":"1731527077459"} 2024-11-13T19:44:37,463 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:44:37,463 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:44:37,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6480 sec 2024-11-13T19:44:37,706 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:44:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:44:37,974 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:37,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:44:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:37,976 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:44:37,977 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:37,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:37,981 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/recovered.edits] 2024-11-13T19:44:37,984 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1e29a479f54a4dba9b49f8b811959da2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/1e29a479f54a4dba9b49f8b811959da2 2024-11-13T19:44:37,985 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/646e5271d48545669425657f30c2967b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/646e5271d48545669425657f30c2967b 2024-11-13T19:44:37,986 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7cc7fd5fa5fb40f4918e1943f5373746 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7cc7fd5fa5fb40f4918e1943f5373746 2024-11-13T19:44:37,987 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7ede5e1fa93f449da2359f8afff4bec2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/A/7ede5e1fa93f449da2359f8afff4bec2 2024-11-13T19:44:37,989 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/403262f2e6cf45e3bcca21aa3ff1f549 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/403262f2e6cf45e3bcca21aa3ff1f549 2024-11-13T19:44:37,990 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7dbed4809135467eb3135413c34c823b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/7dbed4809135467eb3135413c34c823b 2024-11-13T19:44:37,991 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/98c203ea3d81443ab96f624fc053fccc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/98c203ea3d81443ab96f624fc053fccc 2024-11-13T19:44:37,992 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c736f5a2d2c3480a890861541ebfd669 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/B/c736f5a2d2c3480a890861541ebfd669 2024-11-13T19:44:37,994 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/021f91958e8f49ceb032c7bdbf74080d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/021f91958e8f49ceb032c7bdbf74080d 2024-11-13T19:44:37,995 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/28631f274e79416f87adcf10e940e0c6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/28631f274e79416f87adcf10e940e0c6 2024-11-13T19:44:37,996 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c01b5a2877f94ebea485256fd31045b3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/c01b5a2877f94ebea485256fd31045b3 2024-11-13T19:44:37,997 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/f187eeca70de45188b745cd84b36737b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/C/f187eeca70de45188b745cd84b36737b 2024-11-13T19:44:37,999 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/recovered.edits/1330.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd/recovered.edits/1330.seqid 2024-11-13T19:44:38,000 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/6f4ececb48a6d1da14a168d390e345dd 2024-11-13T19:44:38,000 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:44:38,002 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:38,005 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:44:38,007 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:44:38,008 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:38,008 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:44:38,009 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527078008"}]},"ts":"9223372036854775807"} 2024-11-13T19:44:38,010 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:44:38,010 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6f4ececb48a6d1da14a168d390e345dd, NAME => 'TestAcidGuarantees,,1731527047288.6f4ececb48a6d1da14a168d390e345dd.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:44:38,011 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:44:38,011 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527078011"}]},"ts":"9223372036854775807"} 2024-11-13T19:44:38,012 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:44:38,013 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:38,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-13T19:44:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:44:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:44:38,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:38,094 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 254), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=820 (was 944), ProcessCount=11 (was 11), AvailableMemoryMB=477 (was 523) 2024-11-13T19:44:38,104 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=820, ProcessCount=11, AvailableMemoryMB=477 2024-11-13T19:44:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:44:38,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:44:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:38,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:44:38,108 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:38,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-13T19:44:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T19:44:38,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:44:38,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742181_1357 (size=1061) 2024-11-13T19:44:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T19:44:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T19:44:38,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:44:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742182_1358 (size=53) 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7e80afa615107bfa65836ab06e2a3e60, disabling compactions & flushes 2024-11-13T19:44:38,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. after waiting 0 ms 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7e80afa615107bfa65836ab06e2a3e60: Waiting for close lock at 1731527078530Disabling compacts and flushes for region at 1731527078530Disabling writes for close at 1731527078530Writing region close event to WAL at 1731527078530Closed at 1731527078530 2024-11-13T19:44:38,532 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:44:38,532 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731527078532"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527078532"}]},"ts":"1731527078532"} 2024-11-13T19:44:38,534 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:44:38,535 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:44:38,536 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527078535"}]},"ts":"1731527078535"} 2024-11-13T19:44:38,537 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:44:38,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, ASSIGN}] 2024-11-13T19:44:38,538 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, ASSIGN 2024-11-13T19:44:38,539 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:44:38,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:38,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, ASSIGN because future has completed 2024-11-13T19:44:38,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T19:44:38,848 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,848 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:44:38,849 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,849 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:44:38,849 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,849 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,850 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,851 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:38,852 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName A 2024-11-13T19:44:38,852 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:38,852 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:38,852 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,854 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:38,854 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName B 2024-11-13T19:44:38,854 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:38,855 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:38,855 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,856 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:38,856 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName C 2024-11-13T19:44:38,856 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:38,857 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:38,857 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,857 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,857 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,858 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,859 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,859 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,859 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:44:38,861 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,863 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:44:38,863 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 7e80afa615107bfa65836ab06e2a3e60; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60267131, jitterRate=-0.1019497662782669}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:44:38,863 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:38,864 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 7e80afa615107bfa65836ab06e2a3e60: Running coprocessor pre-open hook at 1731527078849Writing region info on filesystem at 1731527078849Initializing all the Stores at 1731527078850 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527078850Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527078850Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527078850Cleaning up temporary data from old regions at 1731527078859 (+9 ms)Running coprocessor post-open hooks at 1731527078863 (+4 ms)Region opened successfully at 1731527078864 (+1 ms) 2024-11-13T19:44:38,869 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., pid=81, masterSystemTime=1731527078844 2024-11-13T19:44:38,872 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,872 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:38,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:38,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:38,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T19:44:38,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 in 183 msec 2024-11-13T19:44:38,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-13T19:44:38,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, ASSIGN in 340 msec 2024-11-13T19:44:38,881 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:44:38,881 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527078881"}]},"ts":"1731527078881"} 2024-11-13T19:44:38,884 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:44:38,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:44:38,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 779 msec 2024-11-13T19:44:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-13T19:44:39,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:39,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b84b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:39,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:39,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:39,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c36da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:39,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:39,249 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b71cf5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:39,250 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:39,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:39,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:44:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:44:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41ad212f 2024-11-13T19:44:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:44:39,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:44:39,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:44:39,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:44:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:44:39,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742183_1359 (size=1097) 2024-11-13T19:44:39,669 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T19:44:39,669 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T19:44:39,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:44:39,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE}] 2024-11-13T19:44:39,676 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE 2024-11-13T19:44:39,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:39,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE because future has completed 2024-11-13T19:44:39,679 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:44:39,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:39,831 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:39,832 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:44:39,832 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 7e80afa615107bfa65836ab06e2a3e60, disabling compactions & flushes 2024-11-13T19:44:39,832 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:39,832 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:39,832 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. after waiting 0 ms 2024-11-13T19:44:39,832 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:39,837 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:44:39,838 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:39,838 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 7e80afa615107bfa65836ab06e2a3e60: Waiting for close lock at 1731527079832Running coprocessor pre-close hooks at 1731527079832Disabling compacts and flushes for region at 1731527079832Disabling writes for close at 1731527079832Writing region close event to WAL at 1731527079832Running coprocessor post-close hooks at 1731527079838 (+6 ms)Closed at 1731527079838 2024-11-13T19:44:39,838 WARN [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 7e80afa615107bfa65836ab06e2a3e60 to self. 2024-11-13T19:44:39,841 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:39,842 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=CLOSED 2024-11-13T19:44:39,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:39,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T19:44:39,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 in 166 msec 2024-11-13T19:44:39,848 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE; state=CLOSED, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=true 2024-11-13T19:44:39,998 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE because future has completed 2024-11-13T19:44:40,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:44:40,156 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,156 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:44:40,156 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,157 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:44:40,157 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,157 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,159 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,160 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:40,160 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName A 2024-11-13T19:44:40,161 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:40,162 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:40,162 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,163 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:40,163 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName B 2024-11-13T19:44:40,163 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:40,163 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:40,164 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,164 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:44:40,164 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e80afa615107bfa65836ab06e2a3e60 columnFamilyName C 2024-11-13T19:44:40,164 DEBUG [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:40,165 INFO [StoreOpener-7e80afa615107bfa65836ab06e2a3e60-1 {}] regionserver.HStore(327): Store=7e80afa615107bfa65836ab06e2a3e60/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:44:40,166 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,166 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,167 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,169 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,170 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,170 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,170 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:44:40,172 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,172 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 7e80afa615107bfa65836ab06e2a3e60; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65698822, jitterRate=-0.021011263132095337}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:44:40,172 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,173 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 7e80afa615107bfa65836ab06e2a3e60: Running coprocessor pre-open hook at 1731527080157Writing region info on filesystem at 1731527080157Initializing all the Stores at 1731527080158 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527080158Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527080158Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527080158Cleaning up temporary data from old regions at 1731527080170 (+12 ms)Running coprocessor post-open hooks at 1731527080172 (+2 ms)Region opened successfully at 1731527080173 (+1 ms) 2024-11-13T19:44:40,174 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., pid=86, masterSystemTime=1731527080152 2024-11-13T19:44:40,175 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,176 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=OPEN, openSeqNum=5, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:44:40,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-13T19:44:40,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 in 178 msec 2024-11-13T19:44:40,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-13T19:44:40,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, REOPEN/MOVE in 505 msec 2024-11-13T19:44:40,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T19:44:40,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-13T19:44:40,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-13T19:44:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:44:40,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:40,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320fb02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1828a07d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11abdc24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79018c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d433d4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,196 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3caa2f8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725f7689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,198 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189d8c52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,199 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635a5f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427fac46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,201 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5864012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a14f801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a741c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,206 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2216475b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,207 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674c5071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab4bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df1ddaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34411e05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108eda1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d074659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dadc450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354663d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,215 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be274e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d74817c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbfc9f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,218 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1376c43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd8484c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c87703e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:44:40,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:44:40,224 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:44:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:44:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:44:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5311a394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:44:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:44:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:44:40,225 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:44:40,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4a8c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:44:40,226 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,227 DEBUG [Thread-1636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,227 DEBUG [Thread-1634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,229 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,233 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:40,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-13T19:44:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:40,237 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:40,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:40,246 DEBUG [Thread-1638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,246 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,247 DEBUG [Thread-1638 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,247 DEBUG [Thread-1638 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,248 DEBUG [Thread-1631 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,248 DEBUG [Thread-1631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,258 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,258 DEBUG [Thread-1640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,259 DEBUG [Thread-1642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,260 DEBUG [Thread-1642 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,260 DEBUG [Thread-1642 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,261 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:44:40,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:44:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:44:40,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:44:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:40,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111335c3f9046ac64ebf96e5ecd85b98a84c_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527080262/Put/seqid=0 2024-11-13T19:44:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527140304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527140305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742184_1360 (size=12154) 2024-11-13T19:44:40,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527140318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111335c3f9046ac64ebf96e5ecd85b98a84c_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335c3f9046ac64ebf96e5ecd85b98a84c_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/792c958a2ec94439816ccb6465d2498b, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:40,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/792c958a2ec94439816ccb6465d2498b is 175, key is test_row_0/A:col10/1731527080262/Put/seqid=0 2024-11-13T19:44:40,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527140322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:40,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527140324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742185_1361 (size=30955) 2024-11-13T19:44:40,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/792c958a2ec94439816ccb6465d2498b 2024-11-13T19:44:40,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/15ba1a50ccc54a01965252006812bbe2 is 50, key is test_row_0/B:col10/1731527080262/Put/seqid=0 2024-11-13T19:44:40,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742186_1362 (size=12001) 2024-11-13T19:44:40,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527140433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527140433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527140444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527140454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527140454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:40,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527140644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527140645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527140664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527140673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527140673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:40,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/15ba1a50ccc54a01965252006812bbe2 2024-11-13T19:44:40,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2310af9f81a849fb94e9331ab1db81fd is 50, key is test_row_0/C:col10/1731527080262/Put/seqid=0 2024-11-13T19:44:40,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:40,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:40,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742187_1363 (size=12001) 2024-11-13T19:44:40,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2310af9f81a849fb94e9331ab1db81fd 2024-11-13T19:44:40,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/792c958a2ec94439816ccb6465d2498b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b 2024-11-13T19:44:40,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T19:44:40,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/15ba1a50ccc54a01965252006812bbe2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2 2024-11-13T19:44:40,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:44:40,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2310af9f81a849fb94e9331ab1db81fd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd 2024-11-13T19:44:40,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:44:40,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7e80afa615107bfa65836ab06e2a3e60 in 695ms, sequenceid=16, compaction requested=false 2024-11-13T19:44:40,957 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:44:40,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:40,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:40,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131104ba9d795a43c4b911ad082e9b58d1_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527080319/Put/seqid=0 2024-11-13T19:44:40,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742188_1364 (size=14594) 2024-11-13T19:44:40,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:40,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131104ba9d795a43c4b911ad082e9b58d1_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131104ba9d795a43c4b911ad082e9b58d1_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:41,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/a9f5e2765248415b842636b6f7b76c02, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:41,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/a9f5e2765248415b842636b6f7b76c02 is 175, key is test_row_0/A:col10/1731527080319/Put/seqid=0 2024-11-13T19:44:41,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527140999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527141005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527141005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742189_1365 (size=39549) 2024-11-13T19:44:41,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/a9f5e2765248415b842636b6f7b76c02 2024-11-13T19:44:41,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/060afce4b3d34e53824f44e091af425f is 50, key is test_row_0/B:col10/1731527080319/Put/seqid=0 2024-11-13T19:44:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527141006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527141007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742190_1366 (size=12001) 2024-11-13T19:44:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527141114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527141114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527141123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527141133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527141134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527141333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527141333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527141345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527141353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527141354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:41,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/060afce4b3d34e53824f44e091af425f 2024-11-13T19:44:41,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/40b7fb07dfe140e48402254660ff264f is 50, key is test_row_0/C:col10/1731527080319/Put/seqid=0 2024-11-13T19:44:41,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742191_1367 (size=12001) 2024-11-13T19:44:41,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527141654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527141654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527141664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527141674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527141674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:41,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:41,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/40b7fb07dfe140e48402254660ff264f 2024-11-13T19:44:41,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/a9f5e2765248415b842636b6f7b76c02 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02 2024-11-13T19:44:41,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02, entries=200, sequenceid=37, filesize=38.6 K 2024-11-13T19:44:41,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/060afce4b3d34e53824f44e091af425f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f 2024-11-13T19:44:41,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:44:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/40b7fb07dfe140e48402254660ff264f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f 2024-11-13T19:44:41,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:44:41,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 7e80afa615107bfa65836ab06e2a3e60 in 931ms, sequenceid=37, compaction requested=false 2024-11-13T19:44:41,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:41,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T19:44:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:41,957 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132b8a6bd0790c403983eda44fb1ad5c06_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527080994/Put/seqid=0 2024-11-13T19:44:41,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742192_1368 (size=12154) 2024-11-13T19:44:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:41,993 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132b8a6bd0790c403983eda44fb1ad5c06_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b8a6bd0790c403983eda44fb1ad5c06_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/f3ba0208735440b5a99b2f27f4daddcb, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/f3ba0208735440b5a99b2f27f4daddcb is 175, key is test_row_0/A:col10/1731527080994/Put/seqid=0 2024-11-13T19:44:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742193_1369 (size=30955) 2024-11-13T19:44:42,014 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/f3ba0208735440b5a99b2f27f4daddcb 2024-11-13T19:44:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cd9cd441a22c40168125f35ffa5682dd is 50, key is test_row_0/B:col10/1731527080994/Put/seqid=0 2024-11-13T19:44:42,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742194_1370 (size=12001) 2024-11-13T19:44:42,039 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cd9cd441a22c40168125f35ffa5682dd 2024-11-13T19:44:42,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2cc5de5709114cef993abb3955d98537 is 50, key is test_row_0/C:col10/1731527080994/Put/seqid=0 2024-11-13T19:44:42,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742195_1371 (size=12001) 2024-11-13T19:44:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:42,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:42,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527142225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527142225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527142226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527142229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527142231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527142344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527142345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527142345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527142353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527142354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2cc5de5709114cef993abb3955d98537 2024-11-13T19:44:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/f3ba0208735440b5a99b2f27f4daddcb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb 2024-11-13T19:44:42,474 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T19:44:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cd9cd441a22c40168125f35ffa5682dd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd 2024-11-13T19:44:42,479 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T19:44:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/2cc5de5709114cef993abb3955d98537 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537 2024-11-13T19:44:42,486 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T19:44:42,487 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7e80afa615107bfa65836ab06e2a3e60 in 530ms, sequenceid=49, compaction requested=true 2024-11-13T19:44:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-13T19:44:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-13T19:44:42,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T19:44:42,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2530 sec 2024-11-13T19:44:42,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.2620 sec 2024-11-13T19:44:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:42,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:42,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cf73b41a8ee54661b03e4b3472d413e7_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:42,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527142586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527142588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742196_1372 (size=12154) 2024-11-13T19:44:42,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:42,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527142595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527142597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cf73b41a8ee54661b03e4b3472d413e7_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cf73b41a8ee54661b03e4b3472d413e7_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527142599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b29d1e1fe281430f8b5aee22dfb20d0c, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:42,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b29d1e1fe281430f8b5aee22dfb20d0c is 175, key is test_row_0/A:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:42,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742197_1373 (size=30955) 2024-11-13T19:44:42,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527142709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527142709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527142725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527142725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527142730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527142924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527142929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527142944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527142945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527142945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:42,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:44:43,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b29d1e1fe281430f8b5aee22dfb20d0c 2024-11-13T19:44:43,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/83c34d34a44f452ba8c733cf3174e02e is 50, key is test_row_0/B:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:43,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742198_1374 (size=12001) 2024-11-13T19:44:43,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/83c34d34a44f452ba8c733cf3174e02e 2024-11-13T19:44:43,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/bad501d03e2947f09218b9ddc8a7cc83 is 50, key is test_row_0/C:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:43,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742199_1375 (size=12001) 2024-11-13T19:44:43,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/bad501d03e2947f09218b9ddc8a7cc83 2024-11-13T19:44:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b29d1e1fe281430f8b5aee22dfb20d0c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c 2024-11-13T19:44:43,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c, entries=150, sequenceid=70, filesize=30.2 K 2024-11-13T19:44:43,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/83c34d34a44f452ba8c733cf3174e02e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e 2024-11-13T19:44:43,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e, entries=150, sequenceid=70, filesize=11.7 K 2024-11-13T19:44:43,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/bad501d03e2947f09218b9ddc8a7cc83 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83 2024-11-13T19:44:43,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83, entries=150, sequenceid=70, filesize=11.7 K 2024-11-13T19:44:43,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 7e80afa615107bfa65836ab06e2a3e60 in 648ms, sequenceid=70, compaction requested=true 2024-11-13T19:44:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:43,204 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:43,205 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:43,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:43,223 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:43,223 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:43,223 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:43,223 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=46.9 K 2024-11-13T19:44:43,223 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:43,223 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:43,223 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:43,223 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=129.3 K 2024-11-13T19:44:43,223 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:43,224 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c] 2024-11-13T19:44:43,227 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ba1a50ccc54a01965252006812bbe2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527080236 2024-11-13T19:44:43,227 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792c958a2ec94439816ccb6465d2498b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527080236 2024-11-13T19:44:43,227 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 060afce4b3d34e53824f44e091af425f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731527080319 2024-11-13T19:44:43,228 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f5e2765248415b842636b6f7b76c02, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731527080307 2024-11-13T19:44:43,228 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9cd441a22c40168125f35ffa5682dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527080994 2024-11-13T19:44:43,229 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ba0208735440b5a99b2f27f4daddcb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527080994 2024-11-13T19:44:43,229 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 83c34d34a44f452ba8c733cf3174e02e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:43,229 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29d1e1fe281430f8b5aee22dfb20d0c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:43,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:43,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:43,258 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#323 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:43,258 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:43,258 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/d1a6485eadf64cd7986084106bcd8a44 is 50, key is test_row_0/B:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:43,278 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f7d8353f00114fa1ab204830c1a3a35e_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:43,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f7d8353f00114fa1ab204830c1a3a35e_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:43,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f7d8353f00114fa1ab204830c1a3a35e_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:43,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113180335cfeb68441497872e8d61c3358b_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527082595/Put/seqid=0 2024-11-13T19:44:43,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742201_1377 (size=4469) 2024-11-13T19:44:43,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742200_1376 (size=12139) 2024-11-13T19:44:43,332 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/d1a6485eadf64cd7986084106bcd8a44 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/d1a6485eadf64cd7986084106bcd8a44 2024-11-13T19:44:43,337 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into d1a6485eadf64cd7986084106bcd8a44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:43,337 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:43,337 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=12, startTime=1731527083205; duration=0sec 2024-11-13T19:44:43,337 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:43,337 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:43,337 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:43,338 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:43,339 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:43,339 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:43,339 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=46.9 K 2024-11-13T19:44:43,339 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2310af9f81a849fb94e9331ab1db81fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527080236 2024-11-13T19:44:43,339 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b7fb07dfe140e48402254660ff264f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731527080319 2024-11-13T19:44:43,340 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc5de5709114cef993abb3955d98537, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731527080994 2024-11-13T19:44:43,340 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bad501d03e2947f09218b9ddc8a7cc83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527143321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,353 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527083311, max timestamp=1731527083348Num uniques 859; 2024-11-13T19:44:43,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:43,354 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5869d9124f764bf9a72576fcb1267244 is 50, key is test_row_0/C:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:43,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527083311, max timestamp=1731527083348Num uniques 859; 2024-11-13T19:44:43,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:43,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527083311, max timestamp=1731527083348Num uniques 859; 2024-11-13T19:44:43,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742202_1378 (size=14594) 2024-11-13T19:44:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742203_1379 (size=12139) 2024-11-13T19:44:43,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527143434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527143436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527143442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527143443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527143465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527143574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527143574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527143574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527143574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527143683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,723 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:43,724 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/184990edc4214fa1ad780814892b45c8 is 175, key is test_row_0/A:col10/1731527082554/Put/seqid=0 2024-11-13T19:44:43,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742204_1380 (size=31093) 2024-11-13T19:44:43,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:43,763 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113180335cfeb68441497872e8d61c3358b_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113180335cfeb68441497872e8d61c3358b_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:43,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8a44d84eb5594e82add3f83b8ca02f5d, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:43,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8a44d84eb5594e82add3f83b8ca02f5d is 175, key is test_row_0/A:col10/1731527082595/Put/seqid=0 2024-11-13T19:44:43,775 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5869d9124f764bf9a72576fcb1267244 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5869d9124f764bf9a72576fcb1267244 2024-11-13T19:44:43,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742205_1381 (size=39549) 2024-11-13T19:44:43,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8a44d84eb5594e82add3f83b8ca02f5d 2024-11-13T19:44:43,783 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 5869d9124f764bf9a72576fcb1267244(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:43,783 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:43,783 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=12, startTime=1731527083205; duration=0sec 2024-11-13T19:44:43,783 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:43,783 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:43,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/1193d9af166b4415bbc1c9dee62bbdbd is 50, key is test_row_0/B:col10/1731527082595/Put/seqid=0 2024-11-13T19:44:43,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742206_1382 (size=12001) 2024-11-13T19:44:43,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/1193d9af166b4415bbc1c9dee62bbdbd 2024-11-13T19:44:43,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/560d2e5ccc504ca8a8efead397eae4af is 50, key is test_row_0/C:col10/1731527082595/Put/seqid=0 2024-11-13T19:44:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527143794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527143794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527143794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527143794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:43,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742207_1383 (size=12001) 2024-11-13T19:44:44,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527144010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527144114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527144114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527144114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527144117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,136 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/184990edc4214fa1ad780814892b45c8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8 2024-11-13T19:44:44,144 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 184990edc4214fa1ad780814892b45c8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:44,144 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:44,144 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=12, startTime=1731527083204; duration=0sec 2024-11-13T19:44:44,144 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:44,144 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:44,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/560d2e5ccc504ca8a8efead397eae4af 2024-11-13T19:44:44,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8a44d84eb5594e82add3f83b8ca02f5d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d 2024-11-13T19:44:44,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d, entries=200, sequenceid=82, filesize=38.6 K 2024-11-13T19:44:44,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/1193d9af166b4415bbc1c9dee62bbdbd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd 2024-11-13T19:44:44,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T19:44:44,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/560d2e5ccc504ca8a8efead397eae4af as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af 2024-11-13T19:44:44,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af, entries=150, sequenceid=82, filesize=11.7 K 2024-11-13T19:44:44,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 7e80afa615107bfa65836ab06e2a3e60 in 987ms, sequenceid=82, compaction requested=false 2024-11-13T19:44:44,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:44:44,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:44,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-13T19:44:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:44,417 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:44,418 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:44,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:44,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T19:44:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:44,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:44,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:44,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343fbbaf7ab674e3b8818f0a89c9cdbde_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:44,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742208_1384 (size=12154) 2024-11-13T19:44:44,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:44,626 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343fbbaf7ab674e3b8818f0a89c9cdbde_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343fbbaf7ab674e3b8818f0a89c9cdbde_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:44,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8589fe70f4984b3b945e0488cd5a7efa, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:44,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8589fe70f4984b3b945e0488cd5a7efa is 175, key is test_row_0/A:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527144646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742209_1385 (size=30955) 2024-11-13T19:44:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527144646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527144664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527144665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,674 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8589fe70f4984b3b945e0488cd5a7efa 2024-11-13T19:44:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527144665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/359666b49aa14669ae8e8a8c52e9d038 is 50, key is test_row_0/B:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742210_1386 (size=12001) 2024-11-13T19:44:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:44,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527144776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527144776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527144796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:44,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:44,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:44,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:44,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527144994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527144998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527145025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:45,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/359666b49aa14669ae8e8a8c52e9d038 2024-11-13T19:44:45,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/0257dde3f9d84ce4893eaeea73b0f5da is 50, key is test_row_0/C:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:45,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742211_1387 (size=12001) 2024-11-13T19:44:45,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/0257dde3f9d84ce4893eaeea73b0f5da 2024-11-13T19:44:45,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/8589fe70f4984b3b945e0488cd5a7efa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa 2024-11-13T19:44:45,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa, entries=150, sequenceid=121, filesize=30.2 K 2024-11-13T19:44:45,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/359666b49aa14669ae8e8a8c52e9d038 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038 2024-11-13T19:44:45,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T19:44:45,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/0257dde3f9d84ce4893eaeea73b0f5da as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da 2024-11-13T19:44:45,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T19:44:45,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 7e80afa615107bfa65836ab06e2a3e60 in 680ms, sequenceid=121, compaction requested=true 2024-11-13T19:44:45,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:45,230 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:45,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:45,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:45,230 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:45,231 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:45,231 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:45,231 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,231 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/d1a6485eadf64cd7986084106bcd8a44, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=35.3 K 2024-11-13T19:44:45,232 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:45,232 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:45,232 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,232 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=99.2 K 2024-11-13T19:44:45,232 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,232 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa] 2024-11-13T19:44:45,232 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a6485eadf64cd7986084106bcd8a44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:45,233 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184990edc4214fa1ad780814892b45c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:45,234 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1193d9af166b4415bbc1c9dee62bbdbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731527082595 2024-11-13T19:44:45,234 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a44d84eb5594e82add3f83b8ca02f5d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731527082585 2024-11-13T19:44:45,234 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 359666b49aa14669ae8e8a8c52e9d038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:45,234 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8589fe70f4984b3b945e0488cd5a7efa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:45,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:45,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:45,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:45,256 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:45,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/ed8aa3428d074235a8a187f4e58dd7c1 is 50, key is test_row_0/B:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:45,262 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:45,305 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ed931615e7d643ef9879498d77331702_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:45,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ed931615e7d643ef9879498d77331702_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:45,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed931615e7d643ef9879498d77331702_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742212_1388 (size=12241) 2024-11-13T19:44:45,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742213_1389 (size=4469) 2024-11-13T19:44:45,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b85b3f30cffa46389db796f92d17cea4_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527085327/Put/seqid=0 2024-11-13T19:44:45,362 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#333 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:45,363 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/299533f32f784ff0b15b22ce9224e9fd is 175, key is test_row_0/A:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:45,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742214_1390 (size=14744) 2024-11-13T19:44:45,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742215_1391 (size=31195) 2024-11-13T19:44:45,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:45,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b85b3f30cffa46389db796f92d17cea4_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b85b3f30cffa46389db796f92d17cea4_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:45,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/48b77df4a16b4cc6aefca78f2a796dfe, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:45,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/48b77df4a16b4cc6aefca78f2a796dfe is 175, key is test_row_0/A:col10/1731527085327/Put/seqid=0 2024-11-13T19:44:45,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742216_1392 (size=39699) 2024-11-13T19:44:45,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/48b77df4a16b4cc6aefca78f2a796dfe 2024-11-13T19:44:45,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/188de52fa056441283091e56433f5178 is 50, key is test_row_0/B:col10/1731527085327/Put/seqid=0 2024-11-13T19:44:45,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742217_1393 (size=12151) 2024-11-13T19:44:45,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527145453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527145455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527145463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:45,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527145584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527145584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527145585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527145674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,684 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 7e80afa615107bfa65836ab06e2a3e60 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527145683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,694 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in 7e80afa615107bfa65836ab06e2a3e60 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,745 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/ed8aa3428d074235a8a187f4e58dd7c1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ed8aa3428d074235a8a187f4e58dd7c1 2024-11-13T19:44:45,750 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into ed8aa3428d074235a8a187f4e58dd7c1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:45,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:45,750 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527085230; duration=0sec 2024-11-13T19:44:45,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:45,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:45,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:45,751 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:45,751 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:45,751 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,751 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5869d9124f764bf9a72576fcb1267244, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=35.3 K 2024-11-13T19:44:45,752 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5869d9124f764bf9a72576fcb1267244, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731527082224 2024-11-13T19:44:45,752 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 560d2e5ccc504ca8a8efead397eae4af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731527082595 2024-11-13T19:44:45,752 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0257dde3f9d84ce4893eaeea73b0f5da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:45,766 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:45,767 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/160498771cd743d695237ae66c5311d9 is 50, key is test_row_0/C:col10/1731527083435/Put/seqid=0 2024-11-13T19:44:45,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742218_1394 (size=12241) 2024-11-13T19:44:45,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527145804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527145806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527145806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:45,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:45,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:45,836 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/299533f32f784ff0b15b22ce9224e9fd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd 2024-11-13T19:44:45,843 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 299533f32f784ff0b15b22ce9224e9fd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:45,843 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:45,844 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=13, startTime=1731527085230; duration=0sec 2024-11-13T19:44:45,844 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:45,844 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:45,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/188de52fa056441283091e56433f5178 2024-11-13T19:44:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/dccf6980e89147d8871e72b2e697a204 is 50, key is test_row_0/C:col10/1731527085327/Put/seqid=0 2024-11-13T19:44:45,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742219_1395 (size=12151) 2024-11-13T19:44:45,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/dccf6980e89147d8871e72b2e697a204 2024-11-13T19:44:45,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/48b77df4a16b4cc6aefca78f2a796dfe as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe 2024-11-13T19:44:45,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe, entries=200, sequenceid=134, filesize=38.8 K 2024-11-13T19:44:45,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/188de52fa056441283091e56433f5178 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178 2024-11-13T19:44:45,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178, entries=150, sequenceid=134, filesize=11.9 K 2024-11-13T19:44:45,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/dccf6980e89147d8871e72b2e697a204 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204 2024-11-13T19:44:45,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204, entries=150, sequenceid=134, filesize=11.9 K 2024-11-13T19:44:45,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T19:44:45,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7e80afa615107bfa65836ab06e2a3e60 in 655ms, sequenceid=134, compaction requested=false 2024-11-13T19:44:45,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:45,984 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:44:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:45,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304b7be5dd1884fb5aa3be08387680b4c_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527085430/Put/seqid=0 2024-11-13T19:44:45,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742220_1396 (size=12304) 2024-11-13T19:44:45,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:46,002 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304b7be5dd1884fb5aa3be08387680b4c_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304b7be5dd1884fb5aa3be08387680b4c_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:46,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:46,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 is 175, key is test_row_0/A:col10/1731527085430/Put/seqid=0 2024-11-13T19:44:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742221_1397 (size=31105) 2024-11-13T19:44:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:46,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527146159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527146160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527146163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,183 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/160498771cd743d695237ae66c5311d9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/160498771cd743d695237ae66c5311d9 2024-11-13T19:44:46,188 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 160498771cd743d695237ae66c5311d9(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:46,188 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:46,188 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527085239; duration=0sec 2024-11-13T19:44:46,188 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:46,188 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527146273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527146283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527146284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,410 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 2024-11-13T19:44:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f935ee54a42458faa00da97ff751586 is 50, key is test_row_0/B:col10/1731527085430/Put/seqid=0 2024-11-13T19:44:46,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742222_1398 (size=12151) 2024-11-13T19:44:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527146494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527146494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527146494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:46,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527146805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527146805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527146805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:46,841 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f935ee54a42458faa00da97ff751586 2024-11-13T19:44:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/82ee7dec8888486fa520f6ef0e37d43a is 50, key is test_row_0/C:col10/1731527085430/Put/seqid=0 2024-11-13T19:44:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742223_1399 (size=12151) 2024-11-13T19:44:46,863 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/82ee7dec8888486fa520f6ef0e37d43a 2024-11-13T19:44:46,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 2024-11-13T19:44:46,877 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4, entries=150, sequenceid=155, filesize=30.4 K 2024-11-13T19:44:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f935ee54a42458faa00da97ff751586 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586 2024-11-13T19:44:46,882 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T19:44:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/82ee7dec8888486fa520f6ef0e37d43a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a 2024-11-13T19:44:46,905 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T19:44:46,909 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7e80afa615107bfa65836ab06e2a3e60 in 925ms, sequenceid=155, compaction requested=true 2024-11-13T19:44:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-13T19:44:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-13T19:44:46,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-13T19:44:46,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5000 sec 2024-11-13T19:44:46,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.5060 sec 2024-11-13T19:44:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:47,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:44:47,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:47,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:47,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139e1267857ac444999700f1df5e107f1a_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:47,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742224_1400 (size=17284) 2024-11-13T19:44:47,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:47,412 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139e1267857ac444999700f1df5e107f1a_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e1267857ac444999700f1df5e107f1a_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:47,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2f635651011549498ad3fa6704d144c1, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:47,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2f635651011549498ad3fa6704d144c1 is 175, key is test_row_0/A:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:47,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742225_1401 (size=48389) 2024-11-13T19:44:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527147435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527147441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527147444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527147553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527147563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527147577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527147704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527147704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,721 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in 7e80afa615107bfa65836ab06e2a3e60 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,722 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in 7e80afa615107bfa65836ab06e2a3e60 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527147774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527147784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527147805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:47,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2f635651011549498ad3fa6704d144c1 2024-11-13T19:44:47,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f017fd49af745489f9a92b319bd6ce2 is 50, key is test_row_0/B:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:47,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742226_1402 (size=12151) 2024-11-13T19:44:47,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f017fd49af745489f9a92b319bd6ce2 2024-11-13T19:44:47,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/413d69aa7a464c5f9fe9dc67c0527ed7 is 50, key is test_row_0/C:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:47,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742227_1403 (size=12151) 2024-11-13T19:44:48,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527148093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527148105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527148125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/413d69aa7a464c5f9fe9dc67c0527ed7 2024-11-13T19:44:48,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2f635651011549498ad3fa6704d144c1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1 2024-11-13T19:44:48,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1, entries=250, sequenceid=172, filesize=47.3 K 2024-11-13T19:44:48,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/5f017fd49af745489f9a92b319bd6ce2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2 2024-11-13T19:44:48,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-13T19:44:48,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/413d69aa7a464c5f9fe9dc67c0527ed7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7 2024-11-13T19:44:48,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7, entries=150, sequenceid=172, filesize=11.9 K 2024-11-13T19:44:48,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7e80afa615107bfa65836ab06e2a3e60 in 1064ms, sequenceid=172, compaction requested=true 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:48,435 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:48,435 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:48,440 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150388 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:48,440 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:48,440 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,440 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=146.9 K 2024-11-13T19:44:48,440 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:48,440 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,440 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1] 2024-11-13T19:44:48,440 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:48,440 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,440 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ed8aa3428d074235a8a187f4e58dd7c1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=47.6 K 2024-11-13T19:44:48,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299533f32f784ff0b15b22ce9224e9fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:48,442 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8aa3428d074235a8a187f4e58dd7c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:48,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b77df4a16b4cc6aefca78f2a796dfe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731527084637 2024-11-13T19:44:48,442 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 188de52fa056441283091e56433f5178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731527084662 2024-11-13T19:44:48,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b92bba4f6f4d4892a4a9cb92e7b7e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731527085430 2024-11-13T19:44:48,443 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f935ee54a42458faa00da97ff751586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731527085430 2024-11-13T19:44:48,443 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f635651011549498ad3fa6704d144c1, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086131 2024-11-13T19:44:48,450 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f017fd49af745489f9a92b319bd6ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086159 2024-11-13T19:44:48,469 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:48,476 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:48,477 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/64e096f6014e42b8963da33d61c629e5 is 50, key is test_row_0/B:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:48,479 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111328090a472b6040db9d95aa293ac0c78d_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:48,482 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111328090a472b6040db9d95aa293ac0c78d_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:48,482 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111328090a472b6040db9d95aa293ac0c78d_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742228_1404 (size=12527) 2024-11-13T19:44:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742229_1405 (size=4469) 2024-11-13T19:44:48,511 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#344 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:48,511 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/054dc2176bd04a8ea413fa64f326c493 is 175, key is test_row_0/A:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:48,515 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/64e096f6014e42b8963da33d61c629e5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64e096f6014e42b8963da33d61c629e5 2024-11-13T19:44:48,525 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 64e096f6014e42b8963da33d61c629e5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:48,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:48,525 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=12, startTime=1731527088435; duration=0sec 2024-11-13T19:44:48,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:48,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:48,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:44:48,527 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:44:48,527 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:48,527 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,527 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/160498771cd743d695237ae66c5311d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=47.6 K 2024-11-13T19:44:48,527 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 160498771cd743d695237ae66c5311d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731527083418 2024-11-13T19:44:48,528 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting dccf6980e89147d8871e72b2e697a204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731527084662 2024-11-13T19:44:48,531 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ee7dec8888486fa520f6ef0e37d43a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731527085430 2024-11-13T19:44:48,532 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 413d69aa7a464c5f9fe9dc67c0527ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086159 2024-11-13T19:44:48,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742230_1406 (size=31481) 2024-11-13T19:44:48,547 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:48,548 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/f424b184b6c74e05b78403ead104fc1c is 50, key is test_row_0/C:col10/1731527087331/Put/seqid=0 2024-11-13T19:44:48,551 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/054dc2176bd04a8ea413fa64f326c493 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493 2024-11-13T19:44:48,557 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 054dc2176bd04a8ea413fa64f326c493(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:48,557 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:48,557 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=12, startTime=1731527088435; duration=0sec 2024-11-13T19:44:48,557 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:48,557 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:48,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742231_1407 (size=12527) 2024-11-13T19:44:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T19:44:48,584 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:48,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-13T19:44:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:48,587 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:48,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:48,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:48,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:48,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111360f7311f0ab540ce99fa58246168320d_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527087439/Put/seqid=0 2024-11-13T19:44:48,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742232_1408 (size=14794) 2024-11-13T19:44:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527148686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527148687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527148689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T19:44:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527148793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:48,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T19:44:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:48,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:48,968 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/f424b184b6c74e05b78403ead104fc1c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/f424b184b6c74e05b78403ead104fc1c 2024-11-13T19:44:48,982 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into f424b184b6c74e05b78403ead104fc1c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:48,982 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:48,982 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=12, startTime=1731527088435; duration=0sec 2024-11-13T19:44:48,982 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:48,983 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527149003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527149034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527149034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T19:44:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:49,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:49,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:49,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:49,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111360f7311f0ab540ce99fa58246168320d_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111360f7311f0ab540ce99fa58246168320d_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:49,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e371f8c9552341cd94a5b28eb8435fd3, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:49,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e371f8c9552341cd94a5b28eb8435fd3 is 175, key is test_row_0/A:col10/1731527087439/Put/seqid=0 2024-11-13T19:44:49,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742233_1409 (size=39749) 2024-11-13T19:44:49,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e371f8c9552341cd94a5b28eb8435fd3 2024-11-13T19:44:49,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cddc656c4c2f4abd98d924a5a7ea7105 is 50, key is test_row_0/B:col10/1731527087439/Put/seqid=0 2024-11-13T19:44:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742234_1410 (size=12151) 2024-11-13T19:44:49,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cddc656c4c2f4abd98d924a5a7ea7105 2024-11-13T19:44:49,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/27680e4522eb45d4afb624a4eb0554a1 is 50, key is test_row_0/C:col10/1731527087439/Put/seqid=0 2024-11-13T19:44:49,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742235_1411 (size=12151) 2024-11-13T19:44:49,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/27680e4522eb45d4afb624a4eb0554a1 2024-11-13T19:44:49,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e371f8c9552341cd94a5b28eb8435fd3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3 2024-11-13T19:44:49,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3, entries=200, sequenceid=191, filesize=38.8 K 2024-11-13T19:44:49,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/cddc656c4c2f4abd98d924a5a7ea7105 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105 2024-11-13T19:44:49,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T19:44:49,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/27680e4522eb45d4afb624a4eb0554a1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1 2024-11-13T19:44:49,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T19:44:49,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7e80afa615107bfa65836ab06e2a3e60 in 535ms, sequenceid=191, compaction requested=false 2024-11-13T19:44:49,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:49,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T19:44:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:49,202 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:49,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111373e0f8f194fe4454a92750aec95bddf5_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527088662/Put/seqid=0 2024-11-13T19:44:49,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742236_1412 (size=12304) 2024-11-13T19:44:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:49,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527149422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527149435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527149438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527149544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527149544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527149564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:49,724 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111373e0f8f194fe4454a92750aec95bddf5_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111373e0f8f194fe4454a92750aec95bddf5_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:49,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c129ce37d15a4ee681a392d208ebb9ce, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:49,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c129ce37d15a4ee681a392d208ebb9ce is 175, key is test_row_0/A:col10/1731527088662/Put/seqid=0 2024-11-13T19:44:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742237_1413 (size=31105) 2024-11-13T19:44:49,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527149754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527149764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527149783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527150074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527150084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527150103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,142 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c129ce37d15a4ee681a392d208ebb9ce 2024-11-13T19:44:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2487ec449ab640c09543feb889429fa6 is 50, key is test_row_0/B:col10/1731527088662/Put/seqid=0 2024-11-13T19:44:50,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742238_1414 (size=12151) 2024-11-13T19:44:50,165 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2487ec449ab640c09543feb889429fa6 2024-11-13T19:44:50,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5fb892011ae2499dac8b4b14451c223a is 50, key is test_row_0/C:col10/1731527088662/Put/seqid=0 2024-11-13T19:44:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742239_1415 (size=12151) 2024-11-13T19:44:50,601 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5fb892011ae2499dac8b4b14451c223a 2024-11-13T19:44:50,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527150584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527150607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527150616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:50,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c129ce37d15a4ee681a392d208ebb9ce as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce 2024-11-13T19:44:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:50,641 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce, entries=150, sequenceid=207, filesize=30.4 K 2024-11-13T19:44:50,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2487ec449ab640c09543feb889429fa6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6 2024-11-13T19:44:50,682 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T19:44:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5fb892011ae2499dac8b4b14451c223a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a 2024-11-13T19:44:50,692 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T19:44:50,693 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7e80afa615107bfa65836ab06e2a3e60 in 1491ms, sequenceid=207, compaction requested=true 2024-11-13T19:44:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-13T19:44:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-13T19:44:50,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T19:44:50,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-11-13T19:44:50,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1170 sec 2024-11-13T19:44:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T19:44:50,744 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:50,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-13T19:44:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T19:44:50,746 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:50,747 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:50,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T19:44:50,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T19:44:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:50,900 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:44:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323e21a1b46ac4b13b7cfe67e16089f07_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527089401/Put/seqid=0 2024-11-13T19:44:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742240_1416 (size=12304) 2024-11-13T19:44:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T19:44:51,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,354 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323e21a1b46ac4b13b7cfe67e16089f07_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e21a1b46ac4b13b7cfe67e16089f07_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2d5e766f30134862a40cf07ed4f8b4db, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2d5e766f30134862a40cf07ed4f8b4db is 175, key is test_row_0/A:col10/1731527089401/Put/seqid=0 2024-11-13T19:44:51,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742241_1417 (size=31105) 2024-11-13T19:44:51,374 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2d5e766f30134862a40cf07ed4f8b4db 2024-11-13T19:44:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T19:44:51,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/ad2af51434b440ebbfc086be5c1ed0e6 is 50, key is test_row_0/B:col10/1731527089401/Put/seqid=0 2024-11-13T19:44:51,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742242_1418 (size=12151) 2024-11-13T19:44:51,415 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/ad2af51434b440ebbfc086be5c1ed0e6 2024-11-13T19:44:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/84ce8f13a07a4d86a3c3d1b531843bad is 50, key is test_row_0/C:col10/1731527089401/Put/seqid=0 2024-11-13T19:44:51,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742243_1419 (size=12151) 2024-11-13T19:44:51,437 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/84ce8f13a07a4d86a3c3d1b531843bad 2024-11-13T19:44:51,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2d5e766f30134862a40cf07ed4f8b4db as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db 2024-11-13T19:44:51,448 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db, entries=150, sequenceid=224, filesize=30.4 K 2024-11-13T19:44:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/ad2af51434b440ebbfc086be5c1ed0e6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6 2024-11-13T19:44:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,453 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T19:44:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/84ce8f13a07a4d86a3c3d1b531843bad as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad 2024-11-13T19:44:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,460 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad, entries=150, sequenceid=224, filesize=11.9 K 2024-11-13T19:44:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,461 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 7e80afa615107bfa65836ab06e2a3e60 in 560ms, sequenceid=224, compaction requested=true 2024-11-13T19:44:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T19:44:51,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-11-13T19:44:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 729 msec 2024-11-13T19:44:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:51,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:51,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:51,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137071739a0aab4bb78677697253f9d154_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:51,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742244_1420 (size=22268) 2024-11-13T19:44:51,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527091842, max timestamp=1731527091858Num uniques 859; 2024-11-13T19:44:51,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527091842, max timestamp=1731527091858Num uniques 859; 2024-11-13T19:44:51,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:51,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:51,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527091842, max timestamp=1731527091858Num uniques 859; 2024-11-13T19:44:51,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T19:44:51,884 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:51,885 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T19:44:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:51,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:51,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:51,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527151921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527151922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527151924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527151924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:51,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527151940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:52,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527152045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527152045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527152045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527152045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527152053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:52,214 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:52,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137071739a0aab4bb78677697253f9d154_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137071739a0aab4bb78677697253f9d154_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:52,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/6164ffb6002444f68fc9736dcd7601b2, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:52,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/6164ffb6002444f68fc9736dcd7601b2 is 175, key is test_row_0/A:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:52,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742245_1421 (size=65673) 2024-11-13T19:44:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527152254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527152254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527152254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527152255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527152278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:52,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527152573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527152574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527152574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527152574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527152603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:52,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/6164ffb6002444f68fc9736dcd7601b2 2024-11-13T19:44:52,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/6e375f27905142e48cf4304dc89a95df is 50, key is test_row_0/B:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:52,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742246_1422 (size=12151) 2024-11-13T19:44:52,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:52,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:53,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/6e375f27905142e48cf4304dc89a95df 2024-11-13T19:44:53,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/482b7013c92742fdb8e9bd932eceed99 is 50, key is test_row_0/C:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:53,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742247_1423 (size=12151) 2024-11-13T19:44:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527153094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527153094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527153094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527153103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:53,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527153134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:53,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:53,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/482b7013c92742fdb8e9bd932eceed99 2024-11-13T19:44:53,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/6164ffb6002444f68fc9736dcd7601b2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2 2024-11-13T19:44:53,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2, entries=350, sequenceid=234, filesize=64.1 K 2024-11-13T19:44:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/6e375f27905142e48cf4304dc89a95df as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df 2024-11-13T19:44:53,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df, entries=150, sequenceid=234, filesize=11.9 K 2024-11-13T19:44:53,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/482b7013c92742fdb8e9bd932eceed99 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99 2024-11-13T19:44:53,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99, entries=150, sequenceid=234, filesize=11.9 K 2024-11-13T19:44:53,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e80afa615107bfa65836ab06e2a3e60 in 1713ms, sequenceid=234, compaction requested=true 2024-11-13T19:44:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:53,508 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:53,508 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:53,510 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 199113 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:53,510 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:53,510 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,510 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=194.4 K 2024-11-13T19:44:53,510 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,510 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2] 2024-11-13T19:44:53,511 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054dc2176bd04a8ea413fa64f326c493, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086159 2024-11-13T19:44:53,511 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:53,511 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:53,511 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,511 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64e096f6014e42b8963da33d61c629e5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=59.7 K 2024-11-13T19:44:53,511 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e371f8c9552341cd94a5b28eb8435fd3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731527087430 2024-11-13T19:44:53,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e096f6014e42b8963da33d61c629e5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086159 2024-11-13T19:44:53,512 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c129ce37d15a4ee681a392d208ebb9ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731527088653 2024-11-13T19:44:53,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting cddc656c4c2f4abd98d924a5a7ea7105, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731527087430 2024-11-13T19:44:53,512 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5e766f30134862a40cf07ed4f8b4db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731527089401 2024-11-13T19:44:53,513 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2487ec449ab640c09543feb889429fa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731527088653 2024-11-13T19:44:53,513 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6164ffb6002444f68fc9736dcd7601b2, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091691 2024-11-13T19:44:53,513 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2af51434b440ebbfc086be5c1ed0e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731527089401 2024-11-13T19:44:53,513 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e375f27905142e48cf4304dc89a95df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091771 2024-11-13T19:44:53,521 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:53,527 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:53,528 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/a9d7a36939da4911b9672e6c68915ad9 is 50, key is test_row_0/B:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:53,530 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111370a90e8b4a964ef4b8e0b9f7dc81dbdc_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:53,532 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111370a90e8b4a964ef4b8e0b9f7dc81dbdc_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:53,532 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111370a90e8b4a964ef4b8e0b9f7dc81dbdc_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742249_1425 (size=4469) 2024-11-13T19:44:53,577 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#359 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:53,578 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/502e3c74db444e4c8b9bc81f6f98ee5e is 175, key is test_row_0/A:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742248_1424 (size=12697) 2024-11-13T19:44:53,586 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/a9d7a36939da4911b9672e6c68915ad9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/a9d7a36939da4911b9672e6c68915ad9 2024-11-13T19:44:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742250_1426 (size=31651) 2024-11-13T19:44:53,593 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/502e3c74db444e4c8b9bc81f6f98ee5e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e 2024-11-13T19:44:53,594 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into a9d7a36939da4911b9672e6c68915ad9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:53,594 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:53,594 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=11, startTime=1731527093508; duration=0sec 2024-11-13T19:44:53,594 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:53,594 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:53,595 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:53,597 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:53,597 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:53,598 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,598 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/f424b184b6c74e05b78403ead104fc1c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=59.7 K 2024-11-13T19:44:53,598 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 502e3c74db444e4c8b9bc81f6f98ee5e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:53,598 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:53,598 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=11, startTime=1731527093508; duration=0sec 2024-11-13T19:44:53,598 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:53,598 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:53,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:44:53,600 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f424b184b6c74e05b78403ead104fc1c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731527086159 2024-11-13T19:44:53,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:53,601 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T19:44:53,601 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 27680e4522eb45d4afb624a4eb0554a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731527087430 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:53,601 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb892011ae2499dac8b4b14451c223a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731527088653 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:53,602 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ce8f13a07a4d86a3c3d1b531843bad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731527089401 2024-11-13T19:44:53,604 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 482b7013c92742fdb8e9bd932eceed99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091771 2024-11-13T19:44:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135d17e20b23d9471fbf1f2b4838d9a5df_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527091938/Put/seqid=0 2024-11-13T19:44:53,640 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:53,641 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/8b863be6fea8438f9d1be53ece8a1ff7 is 50, key is test_row_0/C:col10/1731527091771/Put/seqid=0 2024-11-13T19:44:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742251_1427 (size=12454) 2024-11-13T19:44:53,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742252_1428 (size=12697) 2024-11-13T19:44:53,696 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/8b863be6fea8438f9d1be53ece8a1ff7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/8b863be6fea8438f9d1be53ece8a1ff7 2024-11-13T19:44:53,703 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 8b863be6fea8438f9d1be53ece8a1ff7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:53,703 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:53,703 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=11, startTime=1731527093509; duration=0sec 2024-11-13T19:44:53,703 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:53,703 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:54,080 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135d17e20b23d9471fbf1f2b4838d9a5df_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135d17e20b23d9471fbf1f2b4838d9a5df_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/621b21aa7bc647c0a867f8ababd1c09b, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/621b21aa7bc647c0a867f8ababd1c09b is 175, key is test_row_0/A:col10/1731527091938/Put/seqid=0 2024-11-13T19:44:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742253_1429 (size=31255) 2024-11-13T19:44:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:54,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527154137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527154149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527154151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527154151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527154163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,176 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in 7e80afa615107bfa65836ab06e2a3e60 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:44:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527154263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527154274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527154274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527154274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,505 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/621b21aa7bc647c0a867f8ababd1c09b 2024-11-13T19:44:54,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/788173082a804cdd9211d9fce4e78a28 is 50, key is test_row_0/B:col10/1731527091938/Put/seqid=0 2024-11-13T19:44:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527154495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527154495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527154495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527154495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742254_1430 (size=12301) 2024-11-13T19:44:54,525 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/788173082a804cdd9211d9fce4e78a28 2024-11-13T19:44:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/9235327d56b14b23b7bb97e3ddb6a779 is 50, key is test_row_0/C:col10/1731527091938/Put/seqid=0 2024-11-13T19:44:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742255_1431 (size=12301) 2024-11-13T19:44:54,540 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/9235327d56b14b23b7bb97e3ddb6a779 2024-11-13T19:44:54,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/621b21aa7bc647c0a867f8ababd1c09b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b 2024-11-13T19:44:54,554 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b, entries=150, sequenceid=273, filesize=30.5 K 2024-11-13T19:44:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/788173082a804cdd9211d9fce4e78a28 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28 2024-11-13T19:44:54,580 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T19:44:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/9235327d56b14b23b7bb97e3ddb6a779 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779 2024-11-13T19:44:54,586 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T19:44:54,587 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 7e80afa615107bfa65836ab06e2a3e60 in 987ms, sequenceid=273, compaction requested=false 2024-11-13T19:44:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T19:44:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T19:44:54,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T19:44:54,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6990 sec 2024-11-13T19:44:54,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7060 sec 2024-11-13T19:44:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:54,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:54,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135ba85ab6372e4ca093388441fa903573_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527154883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527154885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742256_1432 (size=12454) 2024-11-13T19:44:54,897 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527154889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,904 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135ba85ab6372e4ca093388441fa903573_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ba85ab6372e4ca093388441fa903573_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:54,905 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/1633371e24fe4a109ba99ebd15fc70a6, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:54,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/1633371e24fe4a109ba99ebd15fc70a6 is 175, key is test_row_0/A:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:54,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527154891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:54,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742257_1433 (size=31255) 2024-11-13T19:44:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527155004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527155004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527155014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527155015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527155214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527155214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527155233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527155233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,314 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/1633371e24fe4a109ba99ebd15fc70a6 2024-11-13T19:44:55,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 is 50, key is test_row_0/B:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:55,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742258_1434 (size=12301) 2024-11-13T19:44:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527155534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527155534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527155544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527155544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:55,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 2024-11-13T19:44:55,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/85ae126ce4ed4220be3da01a5654814a is 50, key is test_row_0/C:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742259_1435 (size=12301) 2024-11-13T19:44:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:44:56,063 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:56,064 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527156054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T19:44:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527156054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:56,067 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:56,067 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:56,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527156064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527156064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:56,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/85ae126ce4ed4220be3da01a5654814a 2024-11-13T19:44:56,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/1633371e24fe4a109ba99ebd15fc70a6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6 2024-11-13T19:44:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:56,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6, entries=150, sequenceid=287, filesize=30.5 K 2024-11-13T19:44:56,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 2024-11-13T19:44:56,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:44:56,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/85ae126ce4ed4220be3da01a5654814a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a 2024-11-13T19:44:56,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:44:56,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7e80afa615107bfa65836ab06e2a3e60 in 1357ms, sequenceid=287, compaction requested=true 2024-11-13T19:44:56,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:56,188 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:56,189 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:56,189 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:56,189 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,189 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=92.0 K 2024-11-13T19:44:56,189 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,189 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6] 2024-11-13T19:44:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:56,190 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:56,190 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 502e3c74db444e4c8b9bc81f6f98ee5e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091771 2024-11-13T19:44:56,191 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 621b21aa7bc647c0a867f8ababd1c09b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731527091921 2024-11-13T19:44:56,191 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:56,191 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:56,191 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,191 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/a9d7a36939da4911b9672e6c68915ad9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.4 K 2024-11-13T19:44:56,191 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1633371e24fe4a109ba99ebd15fc70a6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:56,192 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a9d7a36939da4911b9672e6c68915ad9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091771 2024-11-13T19:44:56,192 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 788173082a804cdd9211d9fce4e78a28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731527091921 2024-11-13T19:44:56,193 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c61bf62ad73e4d8b9e3b7968a0ea87a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:56,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:56,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:56,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:56,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:56,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,208 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:56,208 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/24f55d7ea374477cafb1ba460c2b1852 is 50, key is test_row_0/B:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:56,216 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T19:44:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,235 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113eef053b04754426fa23ee12d869b645e_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,237 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113eef053b04754426fa23ee12d869b645e_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,237 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113eef053b04754426fa23ee12d869b645e_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111349e8dbba16dc41c78ee0ffd918f63fa6_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527094877/Put/seqid=0 2024-11-13T19:44:56,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742260_1436 (size=12949) 2024-11-13T19:44:56,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/24f55d7ea374477cafb1ba460c2b1852 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/24f55d7ea374477cafb1ba460c2b1852 2024-11-13T19:44:56,267 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 24f55d7ea374477cafb1ba460c2b1852(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:56,267 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:56,267 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527096189; duration=0sec 2024-11-13T19:44:56,267 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:56,267 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:56,267 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:56,268 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:56,268 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:56,268 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,268 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/8b863be6fea8438f9d1be53ece8a1ff7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.4 K 2024-11-13T19:44:56,269 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b863be6fea8438f9d1be53ece8a1ff7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731527091771 2024-11-13T19:44:56,269 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9235327d56b14b23b7bb97e3ddb6a779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731527091921 2024-11-13T19:44:56,270 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ae126ce4ed4220be3da01a5654814a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:56,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742262_1438 (size=4469) 2024-11-13T19:44:56,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742261_1437 (size=14994) 2024-11-13T19:44:56,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111349e8dbba16dc41c78ee0ffd918f63fa6_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111349e8dbba16dc41c78ee0ffd918f63fa6_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/370aeb7db1074836b57ba6bf9c77fdda, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/370aeb7db1074836b57ba6bf9c77fdda is 175, key is test_row_0/A:col10/1731527094877/Put/seqid=0 2024-11-13T19:44:56,300 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:56,305 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/11df42531d8448d88eab325ec9d50770 is 50, key is test_row_0/C:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742263_1439 (size=39949) 2024-11-13T19:44:56,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/370aeb7db1074836b57ba6bf9c77fdda 2024-11-13T19:44:56,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2927db554a484a2a89afcaab16b7936f is 50, key is test_row_0/B:col10/1731527094877/Put/seqid=0 2024-11-13T19:44:56,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742264_1440 (size=12949) 2024-11-13T19:44:56,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527156333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742265_1441 (size=12301) 2024-11-13T19:44:56,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2927db554a484a2a89afcaab16b7936f 2024-11-13T19:44:56,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/c271250b74f749bd8fd9162e639d050b is 50, key is test_row_0/C:col10/1731527094877/Put/seqid=0 2024-11-13T19:44:56,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T19:44:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:56,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742266_1442 (size=12301) 2024-11-13T19:44:56,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/c271250b74f749bd8fd9162e639d050b 2024-11-13T19:44:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/370aeb7db1074836b57ba6bf9c77fdda as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda 2024-11-13T19:44:56,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda, entries=200, sequenceid=307, filesize=39.0 K 2024-11-13T19:44:56,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/2927db554a484a2a89afcaab16b7936f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f 2024-11-13T19:44:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-13T19:44:56,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/c271250b74f749bd8fd9162e639d050b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b 2024-11-13T19:44:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-13T19:44:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7e80afa615107bfa65836ab06e2a3e60 in 223ms, sequenceid=307, compaction requested=false 2024-11-13T19:44:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:56,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113819e63d8e64d40dda1c0c8d650cf9c89_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742267_1443 (size=12454) 2024-11-13T19:44:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,492 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,496 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113819e63d8e64d40dda1c0c8d650cf9c89_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113819e63d8e64d40dda1c0c8d650cf9c89_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2a3cffb10f9f409ca8f27f249cc0a2c5, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2a3cffb10f9f409ca8f27f249cc0a2c5 is 175, key is test_row_0/A:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742268_1444 (size=31251) 2024-11-13T19:44:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,505 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2a3cffb10f9f409ca8f27f249cc0a2c5 2024-11-13T19:44:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/f79b2370a91c4557b7cd80a5578f534d is 50, key is test_row_0/B:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742269_1445 (size=12297) 2024-11-13T19:44:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T19:44:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:56,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/f79b2370a91c4557b7cd80a5578f534d 2024-11-13T19:44:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/6bcaf3dc8c3c465fae4324f2626ca283 is 50, key is test_row_0/C:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742270_1446 (size=9857) 2024-11-13T19:44:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/6bcaf3dc8c3c465fae4324f2626ca283 2024-11-13T19:44:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2a3cffb10f9f409ca8f27f249cc0a2c5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5 2024-11-13T19:44:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5, entries=150, sequenceid=321, filesize=30.5 K 2024-11-13T19:44:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/f79b2370a91c4557b7cd80a5578f534d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d 2024-11-13T19:44:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d, entries=150, sequenceid=321, filesize=12.0 K 2024-11-13T19:44:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/6bcaf3dc8c3c465fae4324f2626ca283 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283 2024-11-13T19:44:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283, entries=100, sequenceid=321, filesize=9.6 K 2024-11-13T19:44:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for 7e80afa615107bfa65836ab06e2a3e60 in 97ms, sequenceid=321, compaction requested=true 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:44:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:56,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:44:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T19:44:56,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,686 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T19:44:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:56,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,691 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#369 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:56,692 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c1404b9099b940dbbf7d8a2b86ed84b0 is 175, key is test_row_0/A:col10/1731527094830/Put/seqid=0 2024-11-13T19:44:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd2f72295a314f70a3b174d8d95c451b_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527096676/Put/seqid=0 2024-11-13T19:44:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742271_1447 (size=31903) 2024-11-13T19:44:56,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742272_1448 (size=22618) 2024-11-13T19:44:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:56,743 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd2f72295a314f70a3b174d8d95c451b_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd2f72295a314f70a3b174d8d95c451b_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:56,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e5a076799b7947bc8318141d91564c17, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:56,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e5a076799b7947bc8318141d91564c17 is 175, key is test_row_0/A:col10/1731527096676/Put/seqid=0 2024-11-13T19:44:56,749 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/11df42531d8448d88eab325ec9d50770 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/11df42531d8448d88eab325ec9d50770 2024-11-13T19:44:56,754 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 11df42531d8448d88eab325ec9d50770(size=12.6 K), total size for store is 34.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:56,754 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:56,754 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527096196; duration=0sec 2024-11-13T19:44:56,754 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:44:56,754 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. because compaction request was cancelled 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:56,755 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:56,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742273_1449 (size=66023) 2024-11-13T19:44:56,757 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:56,757 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:56,757 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:56,757 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/24f55d7ea374477cafb1ba460c2b1852, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.7 K 2024-11-13T19:44:56,757 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f55d7ea374477cafb1ba460c2b1852, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:56,758 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e5a076799b7947bc8318141d91564c17 2024-11-13T19:44:56,762 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2927db554a484a2a89afcaab16b7936f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731527094877 2024-11-13T19:44:56,763 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f79b2370a91c4557b7cd80a5578f534d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731527096320 2024-11-13T19:44:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/0a8920d46c21442d886654522ddb9748 is 50, key is test_row_0/B:col10/1731527096676/Put/seqid=0 2024-11-13T19:44:56,780 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:56,781 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/02e403d1d19c4b49a15013f40807f9b2 is 50, key is test_row_0/B:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:56,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742274_1450 (size=12301) 2024-11-13T19:44:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742275_1451 (size=13051) 2024-11-13T19:44:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527096866, max timestamp=1731527096951Num uniques 859; 2024-11-13T19:44:56,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:56,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527096866, max timestamp=1731527096951Num uniques 859; 2024-11-13T19:44:56,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:56,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527096866, max timestamp=1731527096951Num uniques 859; 2024-11-13T19:44:56,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:44:57,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527157096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527157102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527157116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,134 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/c1404b9099b940dbbf7d8a2b86ed84b0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0 2024-11-13T19:44:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527157118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527157118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,141 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into c1404b9099b940dbbf7d8a2b86ed84b0(size=31.2 K), total size for store is 100.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:57,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:57,141 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=13, startTime=1731527096188; duration=0sec 2024-11-13T19:44:57,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:57,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:57,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:57,142 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:57,142 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:57,142 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:57,142 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/11df42531d8448d88eab325ec9d50770, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=34.3 K 2024-11-13T19:44:57,142 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11df42531d8448d88eab325ec9d50770, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:57,143 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c271250b74f749bd8fd9162e639d050b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731527094877 2024-11-13T19:44:57,143 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bcaf3dc8c3c465fae4324f2626ca283, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731527096331 2024-11-13T19:44:57,150 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:57,150 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5b29fa06923f47a89a3151235a808f1b is 50, key is test_row_0/C:col10/1731527096331/Put/seqid=0 2024-11-13T19:44:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742276_1452 (size=13051) 2024-11-13T19:44:57,166 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/5b29fa06923f47a89a3151235a808f1b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5b29fa06923f47a89a3151235a808f1b 2024-11-13T19:44:57,174 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 5b29fa06923f47a89a3151235a808f1b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:57,174 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:57,174 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527096572; duration=0sec 2024-11-13T19:44:57,174 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:57,174 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:57,200 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/0a8920d46c21442d886654522ddb9748 2024-11-13T19:44:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:57,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/198e9e52bf5340399d08a6830beec46b is 50, key is test_row_0/C:col10/1731527096676/Put/seqid=0 2024-11-13T19:44:57,210 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/02e403d1d19c4b49a15013f40807f9b2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/02e403d1d19c4b49a15013f40807f9b2 2024-11-13T19:44:57,216 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 02e403d1d19c4b49a15013f40807f9b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:57,216 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:57,216 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527096572; duration=0sec 2024-11-13T19:44:57,216 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:57,216 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:57,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742277_1453 (size=12301) 2024-11-13T19:44:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527157224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527157234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527157234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527157243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527157243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527157444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527157444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527157444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527157463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527157463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,634 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/198e9e52bf5340399d08a6830beec46b 2024-11-13T19:44:57,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/e5a076799b7947bc8318141d91564c17 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17 2024-11-13T19:44:57,646 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17, entries=350, sequenceid=331, filesize=64.5 K 2024-11-13T19:44:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/0a8920d46c21442d886654522ddb9748 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748 2024-11-13T19:44:57,652 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T19:44:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/198e9e52bf5340399d08a6830beec46b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b 2024-11-13T19:44:57,657 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T19:44:57,658 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e80afa615107bfa65836ab06e2a3e60 in 972ms, sequenceid=331, compaction requested=true 2024-11-13T19:44:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T19:44:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T19:44:57,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T19:44:57,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5920 sec 2024-11-13T19:44:57,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5990 sec 2024-11-13T19:44:57,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:57,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:44:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:57,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139f454ddc93674a9b9295fec437d4f0a6_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:57,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742278_1454 (size=14994) 2024-11-13T19:44:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527157785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527157796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527157800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527157801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527157803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527157931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527157935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527157935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527157935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527158164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527158164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527158164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527158164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:58,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139f454ddc93674a9b9295fec437d4f0a6_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f454ddc93674a9b9295fec437d4f0a6_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:58,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/eb66cfb7bae84a09b4e0dcd5f1656f15, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/eb66cfb7bae84a09b4e0dcd5f1656f15 is 175, key is test_row_0/A:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742279_1455 (size=39949) 2024-11-13T19:44:58,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/eb66cfb7bae84a09b4e0dcd5f1656f15 2024-11-13T19:44:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:44:58,214 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:44:58,215 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:44:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T19:44:58,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:44:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:44:58,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:44:58,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:44:58,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/abf1fb2a52794f5f9b477ed4d39b0e9a is 50, key is test_row_0/B:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742280_1456 (size=12301) 2024-11-13T19:44:58,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/abf1fb2a52794f5f9b477ed4d39b0e9a 2024-11-13T19:44:58,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/b0ebbb2f877448fcb749c77988ff8b81 is 50, key is test_row_0/C:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742281_1457 (size=12301) 2024-11-13T19:44:58,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/b0ebbb2f877448fcb749c77988ff8b81 2024-11-13T19:44:58,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/eb66cfb7bae84a09b4e0dcd5f1656f15 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15 2024-11-13T19:44:58,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15, entries=200, sequenceid=373, filesize=39.0 K 2024-11-13T19:44:58,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/abf1fb2a52794f5f9b477ed4d39b0e9a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a 2024-11-13T19:44:58,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T19:44:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/b0ebbb2f877448fcb749c77988ff8b81 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81 2024-11-13T19:44:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T19:44:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 7e80afa615107bfa65836ab06e2a3e60 in 511ms, sequenceid=373, compaction requested=true 2024-11-13T19:44:58,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:58,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:44:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:44:58,281 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:58,286 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:58,286 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:58,286 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,286 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/02e403d1d19c4b49a15013f40807f9b2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.8 K 2024-11-13T19:44:58,286 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 209075 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:44:58,286 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e403d1d19c4b49a15013f40807f9b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731527096199 2024-11-13T19:44:58,286 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:58,287 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,287 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=204.2 K 2024-11-13T19:44:58,287 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,287 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8920d46c21442d886654522ddb9748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731527096638 2024-11-13T19:44:58,287 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15] 2024-11-13T19:44:58,287 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting abf1fb2a52794f5f9b477ed4d39b0e9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097114 2024-11-13T19:44:58,287 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1404b9099b940dbbf7d8a2b86ed84b0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527094135 2024-11-13T19:44:58,287 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370aeb7db1074836b57ba6bf9c77fdda, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731527094877 2024-11-13T19:44:58,288 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3cffb10f9f409ca8f27f249cc0a2c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731527096320 2024-11-13T19:44:58,288 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a076799b7947bc8318141d91564c17, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731527096523 2024-11-13T19:44:58,288 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb66cfb7bae84a09b4e0dcd5f1656f15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097074 2024-11-13T19:44:58,297 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:58,297 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/64b91913cac04452897475e6c8d94987 is 50, key is test_row_0/B:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,304 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742282_1458 (size=13153) 2024-11-13T19:44:58,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113e88b0bd2a6e448bdb52315853c3be11f_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113e88b0bd2a6e448bdb52315853c3be11f_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e88b0bd2a6e448bdb52315853c3be11f_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,314 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/64b91913cac04452897475e6c8d94987 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64b91913cac04452897475e6c8d94987 2024-11-13T19:44:58,318 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 64b91913cac04452897475e6c8d94987(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:58,318 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,318 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527098281; duration=0sec 2024-11-13T19:44:58,318 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:58,318 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:58,318 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:58,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:58,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:58,320 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,320 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5b29fa06923f47a89a3151235a808f1b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.8 K 2024-11-13T19:44:58,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b29fa06923f47a89a3151235a808f1b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731527096199 2024-11-13T19:44:58,321 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 198e9e52bf5340399d08a6830beec46b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731527096638 2024-11-13T19:44:58,321 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ebbb2f877448fcb749c77988ff8b81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097114 2024-11-13T19:44:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:58,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:44:58,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742283_1459 (size=4469) 2024-11-13T19:44:58,325 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#386 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:58,325 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/92b578aa5d654d71b48a56172c751200 is 175, key is test_row_0/A:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cc4b093265b3477d87386f067015f6f4_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527097796/Put/seqid=0 2024-11-13T19:44:58,351 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:58,352 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/6a2a1192b306476d8735a4704cb73920 is 50, key is test_row_0/C:col10/1731527097766/Put/seqid=0 2024-11-13T19:44:58,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742284_1460 (size=32073) 2024-11-13T19:44:58,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:44:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742286_1462 (size=13153) 2024-11-13T19:44:58,385 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/6a2a1192b306476d8735a4704cb73920 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6a2a1192b306476d8735a4704cb73920 2024-11-13T19:44:58,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742285_1461 (size=14994) 2024-11-13T19:44:58,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:58,390 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 6a2a1192b306476d8735a4704cb73920(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:58,390 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,390 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527098281; duration=0sec 2024-11-13T19:44:58,390 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:58,390 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:58,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cc4b093265b3477d87386f067015f6f4_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cc4b093265b3477d87386f067015f6f4_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:58,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2c356e08ef8c47e8a378895c80881fb9, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2c356e08ef8c47e8a378895c80881fb9 is 175, key is test_row_0/A:col10/1731527097796/Put/seqid=0 2024-11-13T19:44:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742287_1463 (size=39949) 2024-11-13T19:44:58,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2c356e08ef8c47e8a378895c80881fb9 2024-11-13T19:44:58,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c59758f85e344caa9f20f224d12660bc is 50, key is test_row_0/B:col10/1731527097796/Put/seqid=0 2024-11-13T19:44:58,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742288_1464 (size=12301) 2024-11-13T19:44:58,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527098485, max timestamp=1731527098490Num uniques 859; 2024-11-13T19:44:58,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:44:58,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527098485, max timestamp=1731527098490Num uniques 859; 2024-11-13T19:44:58,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:44:58,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527098485, max timestamp=1731527098490Num uniques 859; 2024-11-13T19:44:58,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:44:58,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:44:58,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:58,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:44:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527158526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527158528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527158531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527158545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527158545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:44:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527158673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,771 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/92b578aa5d654d71b48a56172c751200 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200 2024-11-13T19:44:58,775 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 92b578aa5d654d71b48a56172c751200(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:58,775 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,775 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=11, startTime=1731527098280; duration=0sec 2024-11-13T19:44:58,775 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:58,775 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:58,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c59758f85e344caa9f20f224d12660bc 2024-11-13T19:44:58,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/ae663a09e30f4fdd9c9a38fced08a028 is 50, key is test_row_0/C:col10/1731527097796/Put/seqid=0 2024-11-13T19:44:58,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:44:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:44:58,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742289_1465 (size=12301) 2024-11-13T19:44:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:44:58,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/ae663a09e30f4fdd9c9a38fced08a028 2024-11-13T19:44:58,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/2c356e08ef8c47e8a378895c80881fb9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9 2024-11-13T19:44:58,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9, entries=200, sequenceid=384, filesize=39.0 K 2024-11-13T19:44:58,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c59758f85e344caa9f20f224d12660bc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc 2024-11-13T19:44:58,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T19:44:58,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/ae663a09e30f4fdd9c9a38fced08a028 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028 2024-11-13T19:44:58,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T19:44:58,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7e80afa615107bfa65836ab06e2a3e60 in 539ms, sequenceid=384, compaction requested=false 2024-11-13T19:44:58,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:58,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-13T19:44:58,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:58,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:44:58,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139dd5ed025bd3473f9ec25e548ad9b6f7_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:58,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742290_1466 (size=14994) 2024-11-13T19:44:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527158884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527158894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,904 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:58,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139dd5ed025bd3473f9ec25e548ad9b6f7_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dd5ed025bd3473f9ec25e548ad9b6f7_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:58,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06c3c9f739304ec4b475d4eed56dcd9f, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:58,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06c3c9f739304ec4b475d4eed56dcd9f is 175, key is test_row_0/A:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527158899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527158899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527158900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:58,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742291_1467 (size=39949) 2024-11-13T19:44:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:58,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06c3c9f739304ec4b475d4eed56dcd9f 2024-11-13T19:44:58,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/10cbbbdc34d84600911a380c1b3418d2 is 50, key is test_row_0/B:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:58,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742292_1468 (size=12301) 2024-11-13T19:44:58,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/10cbbbdc34d84600911a380c1b3418d2 2024-11-13T19:44:58,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/a8fb1004837f416caaeb3e439375e488 is 50, key is test_row_0/C:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:58,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742293_1469 (size=12301) 2024-11-13T19:44:58,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/a8fb1004837f416caaeb3e439375e488 2024-11-13T19:44:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06c3c9f739304ec4b475d4eed56dcd9f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f 2024-11-13T19:44:58,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f, entries=200, sequenceid=422, filesize=39.0 K 2024-11-13T19:44:58,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/10cbbbdc34d84600911a380c1b3418d2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2 2024-11-13T19:44:58,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2, entries=150, sequenceid=422, filesize=12.0 K 2024-11-13T19:44:58,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/a8fb1004837f416caaeb3e439375e488 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488 2024-11-13T19:44:58,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488, entries=150, sequenceid=422, filesize=12.0 K 2024-11-13T19:44:58,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 7e80afa615107bfa65836ab06e2a3e60 in 107ms, sequenceid=422, compaction requested=true 2024-11-13T19:44:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:58,988 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:58,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:58,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:44:58,989 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,989 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=109.3 K 2024-11-13T19:44:58,989 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f] 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:44:58,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:44:58,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b578aa5d654d71b48a56172c751200, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097114 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:44:58,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:58,989 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:58,990 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:44:58,990 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c356e08ef8c47e8a378895c80881fb9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731527097794 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:44:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:44:58,990 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c3c9f739304ec4b475d4eed56dcd9f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:44:58,991 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:58,991 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:44:58,991 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:58,991 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64b91913cac04452897475e6c8d94987, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.9 K 2024-11-13T19:44:58,993 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b91913cac04452897475e6c8d94987, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097114 2024-11-13T19:44:58,994 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c59758f85e344caa9f20f224d12660bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731527097794 2024-11-13T19:44:58,994 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cbbbdc34d84600911a380c1b3418d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:44:59,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362fcd1b9a6db4db5bfe360a6b7408690_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527098881/Put/seqid=0 2024-11-13T19:44:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:59,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. as already flushing 2024-11-13T19:44:59,030 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:59,040 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:59,041 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/8d2d91b15ec84d69a4795780eea8ec58 is 50, key is test_row_0/B:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:59,047 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132d54712a37514128a3ee49e0aa9475a0_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:59,049 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132d54712a37514128a3ee49e0aa9475a0_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:59,049 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d54712a37514128a3ee49e0aa9475a0_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742294_1470 (size=12454) 2024-11-13T19:44:59,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742296_1472 (size=4469) 2024-11-13T19:44:59,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742295_1471 (size=13255) 2024-11-13T19:44:59,080 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#395 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:59,080 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b04fd56cbe5d4239911825b2e4f5182c is 175, key is test_row_0/A:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:59,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742297_1473 (size=32175) 2024-11-13T19:44:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527099085, max timestamp=1731527099107Num uniques 859; 2024-11-13T19:44:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527099085, max timestamp=1731527099107Num uniques 859; 2024-11-13T19:44:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:59,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527099085, max timestamp=1731527099107Num uniques 859; 2024-11-13T19:44:59,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:44:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527159157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527159170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527159171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527159178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527159204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527159283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527159304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527159304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527159304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:44:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:44:59,487 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362fcd1b9a6db4db5bfe360a6b7408690_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362fcd1b9a6db4db5bfe360a6b7408690_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:44:59,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/de5f07be46484fec907c9d06427831c9, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:44:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/de5f07be46484fec907c9d06427831c9 is 175, key is test_row_0/A:col10/1731527098881/Put/seqid=0 2024-11-13T19:44:59,495 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/8d2d91b15ec84d69a4795780eea8ec58 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/8d2d91b15ec84d69a4795780eea8ec58 2024-11-13T19:44:59,502 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 8d2d91b15ec84d69a4795780eea8ec58(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:59,502 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:59,502 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527098989; duration=0sec 2024-11-13T19:44:59,502 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:44:59,502 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:44:59,502 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:44:59,504 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/b04fd56cbe5d4239911825b2e4f5182c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c 2024-11-13T19:44:59,505 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:44:59,506 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:44:59,506 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:44:59,506 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6a2a1192b306476d8735a4704cb73920, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=36.9 K 2024-11-13T19:44:59,507 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2a1192b306476d8735a4704cb73920, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731527097114 2024-11-13T19:44:59,510 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ae663a09e30f4fdd9c9a38fced08a028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731527097794 2024-11-13T19:44:59,511 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fb1004837f416caaeb3e439375e488, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:44:59,514 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into b04fd56cbe5d4239911825b2e4f5182c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:59,514 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:59,514 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=13, startTime=1731527098988; duration=0sec 2024-11-13T19:44:59,514 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:59,514 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:44:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527159504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,521 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:44:59,522 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/4ed3dec96056492390c88a43cc8921f6 is 50, key is test_row_0/C:col10/1731527098525/Put/seqid=0 2024-11-13T19:44:59,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742298_1474 (size=31255) 2024-11-13T19:44:59,524 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/de5f07be46484fec907c9d06427831c9 2024-11-13T19:44:59,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/863cba288f2e42418ec4379dd630d790 is 50, key is test_row_0/B:col10/1731527098881/Put/seqid=0 2024-11-13T19:44:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527159523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527159524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527159524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742299_1475 (size=13255) 2024-11-13T19:44:59,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742300_1476 (size=12301) 2024-11-13T19:44:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57034 deadline: 1731527159734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527159823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527159844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527159844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:44:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527159844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:44:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:44:59,954 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/4ed3dec96056492390c88a43cc8921f6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/4ed3dec96056492390c88a43cc8921f6 2024-11-13T19:44:59,965 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/863cba288f2e42418ec4379dd630d790 2024-11-13T19:44:59,966 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 4ed3dec96056492390c88a43cc8921f6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:44:59,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:44:59,967 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527098989; duration=0sec 2024-11-13T19:44:59,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:44:59,967 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:44:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/347e6a841ff34206903989e3bbfcc48e is 50, key is test_row_0/C:col10/1731527098881/Put/seqid=0 2024-11-13T19:45:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742301_1477 (size=12301) 2024-11-13T19:45:00,262 INFO [Thread-1636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1636. 2024-11-13T19:45:00,263 DEBUG [Thread-1636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,263 DEBUG [Thread-1636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,263 DEBUG [Thread-1636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,266 INFO [Thread-1634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1634. 2024-11-13T19:45:00,266 DEBUG [Thread-1634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,266 DEBUG [Thread-1634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,267 DEBUG [Thread-1634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,273 INFO [Thread-1638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1638. 2024-11-13T19:45:00,273 DEBUG [Thread-1638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,273 DEBUG [Thread-1638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,273 DEBUG [Thread-1638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,282 INFO [Thread-1642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1642. 2024-11-13T19:45:00,282 DEBUG [Thread-1642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,282 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,282 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,283 INFO [Thread-1640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1640. 2024-11-13T19:45:00,283 DEBUG [Thread-1640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,283 DEBUG [Thread-1640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,283 DEBUG [Thread-1640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57102 deadline: 1731527160344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:45:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57080 deadline: 1731527160364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57070 deadline: 1731527160364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:00,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731527160364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:00,412 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/347e6a841ff34206903989e3bbfcc48e 2024-11-13T19:45:00,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/de5f07be46484fec907c9d06427831c9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9 2024-11-13T19:45:00,422 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9, entries=150, sequenceid=431, filesize=30.5 K 2024-11-13T19:45:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/863cba288f2e42418ec4379dd630d790 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790 2024-11-13T19:45:00,427 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790, entries=150, sequenceid=431, filesize=12.0 K 2024-11-13T19:45:00,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/347e6a841ff34206903989e3bbfcc48e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e 2024-11-13T19:45:00,432 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e, entries=150, sequenceid=431, filesize=12.0 K 2024-11-13T19:45:00,433 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 7e80afa615107bfa65836ab06e2a3e60 in 1444ms, sequenceid=431, compaction requested=false 2024-11-13T19:45:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:45:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:00,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T19:45:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T19:45:00,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T19:45:00,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2160 sec 2024-11-13T19:45:00,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.2230 sec 2024-11-13T19:45:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:00,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:45:00,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527099168, max timestamp=1731527100754Num uniques 1068; 2024-11-13T19:45:00,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527099168, max timestamp=1731527100754Num uniques 1068; 2024-11-13T19:45:00,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T19:45:00,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:45:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:00,759 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-13T19:45:00,759 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:00,759 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,759 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:00,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:00,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393a1c3acb10d41e78a73ceac1ac23f79_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_0/A:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:00,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742302_1478 (size=12454) 2024-11-13T19:45:00,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:00,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111393a1c3acb10d41e78a73ceac1ac23f79_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393a1c3acb10d41e78a73ceac1ac23f79_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:00,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/4b96923486044785b8b2c5a7c30abebc, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:00,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/4b96923486044785b8b2c5a7c30abebc is 175, key is test_row_0/A:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:00,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742303_1479 (size=31255) 2024-11-13T19:45:01,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/4b96923486044785b8b2c5a7c30abebc 2024-11-13T19:45:01,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/841208ab293d40d4bd4d996fdea25ec6 is 50, key is test_row_0/B:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742304_1480 (size=12301) 2024-11-13T19:45:01,365 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-13T19:45:01,366 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:01,366 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,366 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:01,375 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-13T19:45:01,375 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-13T19:45:01,375 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-13T19:45:01,375 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:01,375 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:01,375 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,375 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:01,375 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,375 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,375 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,375 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,375 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:01,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:01,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:01,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:01,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/841208ab293d40d4bd4d996fdea25ec6 2024-11-13T19:45:01,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/86c845698f7548de80c327d74ba98673 is 50, key is test_row_0/C:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:01,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742305_1481 (size=12301) 2024-11-13T19:45:01,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/86c845698f7548de80c327d74ba98673 2024-11-13T19:45:01,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/4b96923486044785b8b2c5a7c30abebc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc 2024-11-13T19:45:01,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc, entries=150, sequenceid=472, filesize=30.5 K 2024-11-13T19:45:01,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/841208ab293d40d4bd4d996fdea25ec6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6 2024-11-13T19:45:01,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T19:45:01,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/86c845698f7548de80c327d74ba98673 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673 2024-11-13T19:45:01,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T19:45:01,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.83 KB/188240, currentSize=26.84 KB/27480 for 7e80afa615107bfa65836ab06e2a3e60 in 884ms, sequenceid=472, compaction requested=true 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:01,642 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7e80afa615107bfa65836ab06e2a3e60:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:01,642 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/A is initiating minor compaction (all files) 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/B is initiating minor compaction (all files) 2024-11-13T19:45:01,643 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/A in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:01,643 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/B in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:01,643 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=92.5 K 2024-11-13T19:45:01,643 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/8d2d91b15ec84d69a4795780eea8ec58, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=37.0 K 2024-11-13T19:45:01,643 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc] 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2d91b15ec84d69a4795780eea8ec58, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:45:01,643 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04fd56cbe5d4239911825b2e4f5182c, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:45:01,644 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 863cba288f2e42418ec4379dd630d790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731527098881 2024-11-13T19:45:01,644 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5f07be46484fec907c9d06427831c9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731527098881 2024-11-13T19:45:01,644 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 841208ab293d40d4bd4d996fdea25ec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731527099168 2024-11-13T19:45:01,644 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b96923486044785b8b2c5a7c30abebc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731527099168 2024-11-13T19:45:01,665 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:01,667 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#B#compaction#403 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:01,667 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/82297c30c8da42edb9d899c90c14bd61 is 50, key is test_row_0/B:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:01,668 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113980b58441b664f75ad918e87604a3433_7e80afa615107bfa65836ab06e2a3e60 store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:01,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742306_1482 (size=13357) 2024-11-13T19:45:01,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113980b58441b664f75ad918e87604a3433_7e80afa615107bfa65836ab06e2a3e60, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:01,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113980b58441b664f75ad918e87604a3433_7e80afa615107bfa65836ab06e2a3e60 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742307_1483 (size=4469) 2024-11-13T19:45:02,080 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/82297c30c8da42edb9d899c90c14bd61 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/82297c30c8da42edb9d899c90c14bd61 2024-11-13T19:45:02,085 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/B of 7e80afa615107bfa65836ab06e2a3e60 into 82297c30c8da42edb9d899c90c14bd61(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:02,085 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:45:02,085 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/B, priority=13, startTime=1731527101642; duration=0sec 2024-11-13T19:45:02,085 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:02,085 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:B 2024-11-13T19:45:02,086 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:02,086 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:02,086 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): 7e80afa615107bfa65836ab06e2a3e60/C is initiating minor compaction (all files) 2024-11-13T19:45:02,087 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7e80afa615107bfa65836ab06e2a3e60/C in TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:02,087 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/4ed3dec96056492390c88a43cc8921f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp, totalSize=37.0 K 2024-11-13T19:45:02,087 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed3dec96056492390c88a43cc8921f6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731527098525 2024-11-13T19:45:02,087 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 347e6a841ff34206903989e3bbfcc48e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731527098881 2024-11-13T19:45:02,088 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c845698f7548de80c327d74ba98673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731527099168 2024-11-13T19:45:02,097 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#C#compaction#405 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:02,098 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/90f8c92fb2f84055afeeb5231a29c3b9 is 50, key is test_row_0/C:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:02,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742308_1484 (size=13357) 2024-11-13T19:45:02,108 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7e80afa615107bfa65836ab06e2a3e60#A#compaction#404 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:02,108 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06654d5794304c33aa9befd13c29a29d is 175, key is test_row_0/A:col10/1731527100754/Put/seqid=0 2024-11-13T19:45:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742309_1485 (size=32277) 2024-11-13T19:45:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:45:02,374 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 735 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2184 rows 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 744 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2211 rows 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 732 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2196 rows 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 738 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2214 rows 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 725 2024-11-13T19:45:02,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2175 rows 2024-11-13T19:45:02,375 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:45:02,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:02,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:02,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:02,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:02,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:45:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:02,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527102380"}]},"ts":"1731527102380"} 2024-11-13T19:45:02,382 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:45:02,382 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:45:02,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:45:02,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, UNASSIGN}] 2024-11-13T19:45:02,384 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, UNASSIGN 2024-11-13T19:45:02,385 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:02,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, UNASSIGN because future has completed 2024-11-13T19:45:02,386 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:45:02,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:02,507 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/90f8c92fb2f84055afeeb5231a29c3b9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/90f8c92fb2f84055afeeb5231a29c3b9 2024-11-13T19:45:02,518 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/06654d5794304c33aa9befd13c29a29d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06654d5794304c33aa9befd13c29a29d 2024-11-13T19:45:02,519 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/C of 7e80afa615107bfa65836ab06e2a3e60 into 90f8c92fb2f84055afeeb5231a29c3b9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:02,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:45:02,519 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/C, priority=13, startTime=1731527101642; duration=0sec 2024-11-13T19:45:02,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:02,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:C 2024-11-13T19:45:02,523 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7e80afa615107bfa65836ab06e2a3e60/A of 7e80afa615107bfa65836ab06e2a3e60 into 06654d5794304c33aa9befd13c29a29d(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:02,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7e80afa615107bfa65836ab06e2a3e60: 2024-11-13T19:45:02,523 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., storeName=7e80afa615107bfa65836ab06e2a3e60/A, priority=13, startTime=1731527101642; duration=0sec 2024-11-13T19:45:02,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:02,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7e80afa615107bfa65836ab06e2a3e60:A 2024-11-13T19:45:02,539 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 7e80afa615107bfa65836ab06e2a3e60, disabling compactions & flushes 2024-11-13T19:45:02,539 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. after waiting 0 ms 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:02,539 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 7e80afa615107bfa65836ab06e2a3e60 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T19:45:02,539 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=A 2024-11-13T19:45:02,540 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:02,540 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=B 2024-11-13T19:45:02,540 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:02,540 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7e80afa615107bfa65836ab06e2a3e60, store=C 2024-11-13T19:45:02,540 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:02,547 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113845fc38f4a4d4b8082ab93e4037f8837_7e80afa615107bfa65836ab06e2a3e60 is 50, key is test_row_1/A:col10/1731527101373/Put/seqid=0 2024-11-13T19:45:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742310_1486 (size=9914) 2024-11-13T19:45:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:02,967 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:02,971 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113845fc38f4a4d4b8082ab93e4037f8837_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113845fc38f4a4d4b8082ab93e4037f8837_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:02,972 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/886ef25696bf4c1bbe8788e8324b6146, store: [table=TestAcidGuarantees family=A region=7e80afa615107bfa65836ab06e2a3e60] 2024-11-13T19:45:02,973 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/886ef25696bf4c1bbe8788e8324b6146 is 175, key is test_row_1/A:col10/1731527101373/Put/seqid=0 2024-11-13T19:45:02,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742311_1487 (size=22561) 2024-11-13T19:45:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:03,379 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=482, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/886ef25696bf4c1bbe8788e8324b6146 2024-11-13T19:45:03,390 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c1cbdd9373164e5e8c6f35431cd1b546 is 50, key is test_row_1/B:col10/1731527101373/Put/seqid=0 2024-11-13T19:45:03,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742312_1488 (size=9857) 2024-11-13T19:45:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:03,795 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c1cbdd9373164e5e8c6f35431cd1b546 2024-11-13T19:45:03,800 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/74db8c6d7143489ba6bc171228e93109 is 50, key is test_row_1/C:col10/1731527101373/Put/seqid=0 2024-11-13T19:45:03,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742313_1489 (size=9857) 2024-11-13T19:45:04,204 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/74db8c6d7143489ba6bc171228e93109 2024-11-13T19:45:04,208 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/A/886ef25696bf4c1bbe8788e8324b6146 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/886ef25696bf4c1bbe8788e8324b6146 2024-11-13T19:45:04,212 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/886ef25696bf4c1bbe8788e8324b6146, entries=100, sequenceid=482, filesize=22.0 K 2024-11-13T19:45:04,212 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/B/c1cbdd9373164e5e8c6f35431cd1b546 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c1cbdd9373164e5e8c6f35431cd1b546 2024-11-13T19:45:04,216 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c1cbdd9373164e5e8c6f35431cd1b546, entries=100, sequenceid=482, filesize=9.6 K 2024-11-13T19:45:04,216 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/.tmp/C/74db8c6d7143489ba6bc171228e93109 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/74db8c6d7143489ba6bc171228e93109 2024-11-13T19:45:04,219 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/74db8c6d7143489ba6bc171228e93109, entries=100, sequenceid=482, filesize=9.6 K 2024-11-13T19:45:04,220 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7e80afa615107bfa65836ab06e2a3e60 in 1681ms, sequenceid=482, compaction requested=false 2024-11-13T19:45:04,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc] to archive 2024-11-13T19:45:04,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:04,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/792c958a2ec94439816ccb6465d2498b 2024-11-13T19:45:04,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/a9f5e2765248415b842636b6f7b76c02 2024-11-13T19:45:04,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/f3ba0208735440b5a99b2f27f4daddcb 2024-11-13T19:45:04,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/184990edc4214fa1ad780814892b45c8 2024-11-13T19:45:04,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b29d1e1fe281430f8b5aee22dfb20d0c 2024-11-13T19:45:04,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8a44d84eb5594e82add3f83b8ca02f5d 2024-11-13T19:45:04,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/299533f32f784ff0b15b22ce9224e9fd 2024-11-13T19:45:04,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/8589fe70f4984b3b945e0488cd5a7efa 2024-11-13T19:45:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/48b77df4a16b4cc6aefca78f2a796dfe 2024-11-13T19:45:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c4b92bba4f6f4d4892a4a9cb92e7b7e4 2024-11-13T19:45:04,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2f635651011549498ad3fa6704d144c1 2024-11-13T19:45:04,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/054dc2176bd04a8ea413fa64f326c493 2024-11-13T19:45:04,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e371f8c9552341cd94a5b28eb8435fd3 2024-11-13T19:45:04,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c129ce37d15a4ee681a392d208ebb9ce 2024-11-13T19:45:04,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2d5e766f30134862a40cf07ed4f8b4db 2024-11-13T19:45:04,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/6164ffb6002444f68fc9736dcd7601b2 2024-11-13T19:45:04,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/502e3c74db444e4c8b9bc81f6f98ee5e 2024-11-13T19:45:04,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/621b21aa7bc647c0a867f8ababd1c09b 2024-11-13T19:45:04,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/c1404b9099b940dbbf7d8a2b86ed84b0 2024-11-13T19:45:04,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/1633371e24fe4a109ba99ebd15fc70a6 2024-11-13T19:45:04,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/370aeb7db1074836b57ba6bf9c77fdda 2024-11-13T19:45:04,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2a3cffb10f9f409ca8f27f249cc0a2c5 2024-11-13T19:45:04,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/e5a076799b7947bc8318141d91564c17 2024-11-13T19:45:04,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/eb66cfb7bae84a09b4e0dcd5f1656f15 2024-11-13T19:45:04,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/92b578aa5d654d71b48a56172c751200 2024-11-13T19:45:04,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/2c356e08ef8c47e8a378895c80881fb9 2024-11-13T19:45:04,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06c3c9f739304ec4b475d4eed56dcd9f 2024-11-13T19:45:04,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/b04fd56cbe5d4239911825b2e4f5182c 2024-11-13T19:45:04,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/de5f07be46484fec907c9d06427831c9 2024-11-13T19:45:04,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/4b96923486044785b8b2c5a7c30abebc 2024-11-13T19:45:04,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/d1a6485eadf64cd7986084106bcd8a44, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ed8aa3428d074235a8a187f4e58dd7c1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64e096f6014e42b8963da33d61c629e5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/a9d7a36939da4911b9672e6c68915ad9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/24f55d7ea374477cafb1ba460c2b1852, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/02e403d1d19c4b49a15013f40807f9b2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64b91913cac04452897475e6c8d94987, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/8d2d91b15ec84d69a4795780eea8ec58, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6] to archive 2024-11-13T19:45:04,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:04,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/15ba1a50ccc54a01965252006812bbe2 2024-11-13T19:45:04,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/060afce4b3d34e53824f44e091af425f 2024-11-13T19:45:04,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cd9cd441a22c40168125f35ffa5682dd 2024-11-13T19:45:04,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/d1a6485eadf64cd7986084106bcd8a44 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/d1a6485eadf64cd7986084106bcd8a44 2024-11-13T19:45:04,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/83c34d34a44f452ba8c733cf3174e02e 2024-11-13T19:45:04,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/1193d9af166b4415bbc1c9dee62bbdbd 2024-11-13T19:45:04,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ed8aa3428d074235a8a187f4e58dd7c1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ed8aa3428d074235a8a187f4e58dd7c1 2024-11-13T19:45:04,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/359666b49aa14669ae8e8a8c52e9d038 2024-11-13T19:45:04,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/188de52fa056441283091e56433f5178 2024-11-13T19:45:04,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f935ee54a42458faa00da97ff751586 2024-11-13T19:45:04,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64e096f6014e42b8963da33d61c629e5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64e096f6014e42b8963da33d61c629e5 2024-11-13T19:45:04,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/5f017fd49af745489f9a92b319bd6ce2 2024-11-13T19:45:04,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/cddc656c4c2f4abd98d924a5a7ea7105 2024-11-13T19:45:04,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2487ec449ab640c09543feb889429fa6 2024-11-13T19:45:04,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/ad2af51434b440ebbfc086be5c1ed0e6 2024-11-13T19:45:04,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/a9d7a36939da4911b9672e6c68915ad9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/a9d7a36939da4911b9672e6c68915ad9 2024-11-13T19:45:04,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/6e375f27905142e48cf4304dc89a95df 2024-11-13T19:45:04,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/788173082a804cdd9211d9fce4e78a28 2024-11-13T19:45:04,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/24f55d7ea374477cafb1ba460c2b1852 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/24f55d7ea374477cafb1ba460c2b1852 2024-11-13T19:45:04,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c61bf62ad73e4d8b9e3b7968a0ea87a8 2024-11-13T19:45:04,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/2927db554a484a2a89afcaab16b7936f 2024-11-13T19:45:04,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/02e403d1d19c4b49a15013f40807f9b2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/02e403d1d19c4b49a15013f40807f9b2 2024-11-13T19:45:04,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/f79b2370a91c4557b7cd80a5578f534d 2024-11-13T19:45:04,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/0a8920d46c21442d886654522ddb9748 2024-11-13T19:45:04,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64b91913cac04452897475e6c8d94987 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/64b91913cac04452897475e6c8d94987 2024-11-13T19:45:04,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/abf1fb2a52794f5f9b477ed4d39b0e9a 2024-11-13T19:45:04,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c59758f85e344caa9f20f224d12660bc 2024-11-13T19:45:04,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/8d2d91b15ec84d69a4795780eea8ec58 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/8d2d91b15ec84d69a4795780eea8ec58 2024-11-13T19:45:04,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/10cbbbdc34d84600911a380c1b3418d2 2024-11-13T19:45:04,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/863cba288f2e42418ec4379dd630d790 2024-11-13T19:45:04,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/841208ab293d40d4bd4d996fdea25ec6 2024-11-13T19:45:04,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5869d9124f764bf9a72576fcb1267244, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/160498771cd743d695237ae66c5311d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/f424b184b6c74e05b78403ead104fc1c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/8b863be6fea8438f9d1be53ece8a1ff7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/11df42531d8448d88eab325ec9d50770, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5b29fa06923f47a89a3151235a808f1b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6a2a1192b306476d8735a4704cb73920, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/4ed3dec96056492390c88a43cc8921f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673] to archive 2024-11-13T19:45:04,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:04,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2310af9f81a849fb94e9331ab1db81fd 2024-11-13T19:45:04,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/40b7fb07dfe140e48402254660ff264f 2024-11-13T19:45:04,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/2cc5de5709114cef993abb3955d98537 2024-11-13T19:45:04,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5869d9124f764bf9a72576fcb1267244 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5869d9124f764bf9a72576fcb1267244 2024-11-13T19:45:04,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/bad501d03e2947f09218b9ddc8a7cc83 2024-11-13T19:45:04,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/560d2e5ccc504ca8a8efead397eae4af 2024-11-13T19:45:04,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/160498771cd743d695237ae66c5311d9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/160498771cd743d695237ae66c5311d9 2024-11-13T19:45:04,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/0257dde3f9d84ce4893eaeea73b0f5da 2024-11-13T19:45:04,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/dccf6980e89147d8871e72b2e697a204 2024-11-13T19:45:04,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/82ee7dec8888486fa520f6ef0e37d43a 2024-11-13T19:45:04,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/f424b184b6c74e05b78403ead104fc1c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/f424b184b6c74e05b78403ead104fc1c 2024-11-13T19:45:04,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/413d69aa7a464c5f9fe9dc67c0527ed7 2024-11-13T19:45:04,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/27680e4522eb45d4afb624a4eb0554a1 2024-11-13T19:45:04,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5fb892011ae2499dac8b4b14451c223a 2024-11-13T19:45:04,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/84ce8f13a07a4d86a3c3d1b531843bad 2024-11-13T19:45:04,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/8b863be6fea8438f9d1be53ece8a1ff7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/8b863be6fea8438f9d1be53ece8a1ff7 2024-11-13T19:45:04,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/482b7013c92742fdb8e9bd932eceed99 2024-11-13T19:45:04,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/9235327d56b14b23b7bb97e3ddb6a779 2024-11-13T19:45:04,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/11df42531d8448d88eab325ec9d50770 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/11df42531d8448d88eab325ec9d50770 2024-11-13T19:45:04,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/85ae126ce4ed4220be3da01a5654814a 2024-11-13T19:45:04,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/c271250b74f749bd8fd9162e639d050b 2024-11-13T19:45:04,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5b29fa06923f47a89a3151235a808f1b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/5b29fa06923f47a89a3151235a808f1b 2024-11-13T19:45:04,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6bcaf3dc8c3c465fae4324f2626ca283 2024-11-13T19:45:04,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/198e9e52bf5340399d08a6830beec46b 2024-11-13T19:45:04,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6a2a1192b306476d8735a4704cb73920 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/6a2a1192b306476d8735a4704cb73920 2024-11-13T19:45:04,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/b0ebbb2f877448fcb749c77988ff8b81 2024-11-13T19:45:04,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/ae663a09e30f4fdd9c9a38fced08a028 2024-11-13T19:45:04,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/4ed3dec96056492390c88a43cc8921f6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/4ed3dec96056492390c88a43cc8921f6 2024-11-13T19:45:04,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/a8fb1004837f416caaeb3e439375e488 2024-11-13T19:45:04,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/347e6a841ff34206903989e3bbfcc48e 2024-11-13T19:45:04,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/86c845698f7548de80c327d74ba98673 2024-11-13T19:45:04,314 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits/485.seqid, newMaxSeqId=485, maxSeqId=4 2024-11-13T19:45:04,315 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. 2024-11-13T19:45:04,315 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 7e80afa615107bfa65836ab06e2a3e60: Waiting for close lock at 1731527102539Running coprocessor pre-close hooks at 1731527102539Disabling compacts and flushes for region at 1731527102539Disabling writes for close at 1731527102539Obtaining lock to block concurrent updates at 1731527102539Preparing flush snapshotting stores in 7e80afa615107bfa65836ab06e2a3e60 at 1731527102539Finished memstore snapshotting TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731527102540 (+1 ms)Flushing stores of TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60. at 1731527102540Flushing 7e80afa615107bfa65836ab06e2a3e60/A: creating writer at 1731527102540Flushing mob file 7e80afa615107bfa65836ab06e2a3e60/A: appending metadata at 1731527102546 (+6 ms)Flushing mob file 7e80afa615107bfa65836ab06e2a3e60/A: closing flushed file at 1731527102546Flushing 7e80afa615107bfa65836ab06e2a3e60/A: appending metadata at 1731527102972 (+426 ms)Flushing 7e80afa615107bfa65836ab06e2a3e60/A: closing flushed file at 1731527102972Flushing 7e80afa615107bfa65836ab06e2a3e60/B: creating writer at 1731527103384 (+412 ms)Flushing 7e80afa615107bfa65836ab06e2a3e60/B: appending metadata at 1731527103390 (+6 ms)Flushing 7e80afa615107bfa65836ab06e2a3e60/B: closing flushed file at 1731527103390Flushing 7e80afa615107bfa65836ab06e2a3e60/C: creating writer at 1731527103798 (+408 ms)Flushing 7e80afa615107bfa65836ab06e2a3e60/C: appending metadata at 1731527103800 (+2 ms)Flushing 7e80afa615107bfa65836ab06e2a3e60/C: closing flushed file at 1731527103800Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8a098da: reopening flushed file at 1731527104208 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33cb2de4: reopening flushed file at 1731527104212 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e6ce234: reopening flushed file at 1731527104216 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7e80afa615107bfa65836ab06e2a3e60 in 1681ms, sequenceid=482, compaction requested=false at 1731527104220 (+4 ms)Writing region close event to WAL at 1731527104312 (+92 ms)Running coprocessor post-close hooks at 1731527104315 (+3 ms)Closed at 1731527104315 2024-11-13T19:45:04,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=7e80afa615107bfa65836ab06e2a3e60, regionState=CLOSED 2024-11-13T19:45:04,320 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:04,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T19:45:04,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 7e80afa615107bfa65836ab06e2a3e60, server=54d0784d1943,36227,1731526989835 in 1.9520 sec 2024-11-13T19:45:04,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-13T19:45:04,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7e80afa615107bfa65836ab06e2a3e60, UNASSIGN in 1.9560 sec 2024-11-13T19:45:04,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T19:45:04,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-13T19:45:04,346 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527104346"}]},"ts":"1731527104346"} 2024-11-13T19:45:04,349 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:45:04,349 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:45:04,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9730 sec 2024-11-13T19:45:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:45:04,524 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:04,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:45:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,527 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:45:04,528 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,533 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,535 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits] 2024-11-13T19:45:04,537 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06654d5794304c33aa9befd13c29a29d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/06654d5794304c33aa9befd13c29a29d 2024-11-13T19:45:04,538 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/886ef25696bf4c1bbe8788e8324b6146 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/A/886ef25696bf4c1bbe8788e8324b6146 2024-11-13T19:45:04,540 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/82297c30c8da42edb9d899c90c14bd61 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/82297c30c8da42edb9d899c90c14bd61 2024-11-13T19:45:04,541 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c1cbdd9373164e5e8c6f35431cd1b546 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/B/c1cbdd9373164e5e8c6f35431cd1b546 2024-11-13T19:45:04,543 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/74db8c6d7143489ba6bc171228e93109 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/74db8c6d7143489ba6bc171228e93109 2024-11-13T19:45:04,543 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/90f8c92fb2f84055afeeb5231a29c3b9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/C/90f8c92fb2f84055afeeb5231a29c3b9 2024-11-13T19:45:04,546 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits/485.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60/recovered.edits/485.seqid 2024-11-13T19:45:04,546 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,546 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:45:04,547 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:45:04,547 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:45:04,550 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304b7be5dd1884fb5aa3be08387680b4c_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304b7be5dd1884fb5aa3be08387680b4c_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,551 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131104ba9d795a43c4b911ad082e9b58d1_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131104ba9d795a43c4b911ad082e9b58d1_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113180335cfeb68441497872e8d61c3358b_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113180335cfeb68441497872e8d61c3358b_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e21a1b46ac4b13b7cfe67e16089f07_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e21a1b46ac4b13b7cfe67e16089f07_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b8a6bd0790c403983eda44fb1ad5c06_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b8a6bd0790c403983eda44fb1ad5c06_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,554 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335c3f9046ac64ebf96e5ecd85b98a84c_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111335c3f9046ac64ebf96e5ecd85b98a84c_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343fbbaf7ab674e3b8818f0a89c9cdbde_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343fbbaf7ab674e3b8818f0a89c9cdbde_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,556 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111349e8dbba16dc41c78ee0ffd918f63fa6_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111349e8dbba16dc41c78ee0ffd918f63fa6_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ba85ab6372e4ca093388441fa903573_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ba85ab6372e4ca093388441fa903573_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135d17e20b23d9471fbf1f2b4838d9a5df_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135d17e20b23d9471fbf1f2b4838d9a5df_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111360f7311f0ab540ce99fa58246168320d_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111360f7311f0ab540ce99fa58246168320d_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362fcd1b9a6db4db5bfe360a6b7408690_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362fcd1b9a6db4db5bfe360a6b7408690_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137071739a0aab4bb78677697253f9d154_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137071739a0aab4bb78677697253f9d154_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111373e0f8f194fe4454a92750aec95bddf5_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111373e0f8f194fe4454a92750aec95bddf5_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113819e63d8e64d40dda1c0c8d650cf9c89_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113819e63d8e64d40dda1c0c8d650cf9c89_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113845fc38f4a4d4b8082ab93e4037f8837_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113845fc38f4a4d4b8082ab93e4037f8837_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393a1c3acb10d41e78a73ceac1ac23f79_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111393a1c3acb10d41e78a73ceac1ac23f79_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dd5ed025bd3473f9ec25e548ad9b6f7_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139dd5ed025bd3473f9ec25e548ad9b6f7_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e1267857ac444999700f1df5e107f1a_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139e1267857ac444999700f1df5e107f1a_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,566 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f454ddc93674a9b9295fec437d4f0a6_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f454ddc93674a9b9295fec437d4f0a6_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b85b3f30cffa46389db796f92d17cea4_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b85b3f30cffa46389db796f92d17cea4_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,568 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd2f72295a314f70a3b174d8d95c451b_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd2f72295a314f70a3b174d8d95c451b_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,569 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cc4b093265b3477d87386f067015f6f4_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cc4b093265b3477d87386f067015f6f4_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cf73b41a8ee54661b03e4b3472d413e7_7e80afa615107bfa65836ab06e2a3e60 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cf73b41a8ee54661b03e4b3472d413e7_7e80afa615107bfa65836ab06e2a3e60 2024-11-13T19:45:04,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:45:04,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,575 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:45:04,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:45:04,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:45:04,578 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527104578"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:04,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:45:04,580 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7e80afa615107bfa65836ab06e2a3e60, NAME => 'TestAcidGuarantees,,1731527078106.7e80afa615107bfa65836ab06e2a3e60.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:45:04,580 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:45:04,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527104580"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:04,581 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:45:04,582 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-13T19:45:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:45:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:45:04,634 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:04,651 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=463 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=804 (was 820), ProcessCount=11 (was 11), AvailableMemoryMB=262 (was 477) 2024-11-13T19:45:04,659 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=804, ProcessCount=11, AvailableMemoryMB=261 2024-11-13T19:45:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:45:04,661 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:45:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:04,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:45:04,663 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:04,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-13T19:45:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:04,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:45:04,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742314_1490 (size=1061) 2024-11-13T19:45:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:05,070 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:45:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742315_1491 (size=53) 2024-11-13T19:45:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:05,475 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:45:05,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e12f682899aa55a4d182e84dc252f667, disabling compactions & flushes 2024-11-13T19:45:05,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. after waiting 0 ms 2024-11-13T19:45:05,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e12f682899aa55a4d182e84dc252f667: Waiting for close lock at 1731527105476Disabling compacts and flushes for region at 1731527105476Disabling writes for close at 1731527105476Writing region close event to WAL at 1731527105476Closed at 1731527105476 2024-11-13T19:45:05,477 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:45:05,477 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731527105477"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527105477"}]},"ts":"1731527105477"} 2024-11-13T19:45:05,478 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:45:05,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:45:05,479 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527105479"}]},"ts":"1731527105479"} 2024-11-13T19:45:05,480 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:45:05,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, ASSIGN}] 2024-11-13T19:45:05,481 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, ASSIGN 2024-11-13T19:45:05,482 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:45:05,632 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e12f682899aa55a4d182e84dc252f667, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:05,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, ASSIGN because future has completed 2024-11-13T19:45:05,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:05,789 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,789 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:45:05,790 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,790 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:45:05,790 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,790 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,791 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,792 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:05,792 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e12f682899aa55a4d182e84dc252f667 columnFamilyName A 2024-11-13T19:45:05,792 DEBUG [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:05,793 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(327): Store=e12f682899aa55a4d182e84dc252f667/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:05,793 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,794 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:05,794 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e12f682899aa55a4d182e84dc252f667 columnFamilyName B 2024-11-13T19:45:05,794 DEBUG [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:05,794 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(327): Store=e12f682899aa55a4d182e84dc252f667/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:05,794 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,795 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:05,795 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e12f682899aa55a4d182e84dc252f667 columnFamilyName C 2024-11-13T19:45:05,795 DEBUG [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:05,795 INFO [StoreOpener-e12f682899aa55a4d182e84dc252f667-1 {}] regionserver.HStore(327): Store=e12f682899aa55a4d182e84dc252f667/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:05,796 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,796 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,796 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,797 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,797 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,797 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,798 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:45:05,799 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,801 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:45:05,801 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened e12f682899aa55a4d182e84dc252f667; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73184232, jitterRate=0.09053003787994385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:45:05,801 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:05,802 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for e12f682899aa55a4d182e84dc252f667: Running coprocessor pre-open hook at 1731527105790Writing region info on filesystem at 1731527105790Initializing all the Stores at 1731527105791 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527105791Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527105791Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527105791Cleaning up temporary data from old regions at 1731527105797 (+6 ms)Running coprocessor post-open hooks at 1731527105801 (+4 ms)Region opened successfully at 1731527105802 (+1 ms) 2024-11-13T19:45:05,802 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., pid=108, masterSystemTime=1731527105786 2024-11-13T19:45:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:05,804 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,804 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:05,804 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e12f682899aa55a4d182e84dc252f667, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:05,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:05,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T19:45:05,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 in 172 msec 2024-11-13T19:45:05,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-13T19:45:05,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, ASSIGN in 327 msec 2024-11-13T19:45:05,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:45:05,810 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527105810"}]},"ts":"1731527105810"} 2024-11-13T19:45:05,811 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:45:05,811 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:45:05,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-13T19:45:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-13T19:45:06,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:06,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27239e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,827 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fc977f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,830 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6c8a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:45:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:45:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@168d10c5 2024-11-13T19:45:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:45:06,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:45:06,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168cafba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,840 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c933234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58be9ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a136c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a311b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585bff85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@489dbb88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,848 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7436e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,849 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34db6dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791548b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfb075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790128a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a2cf1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e38bfb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41055e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79925993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7460faa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17186f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@601ca7fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f89ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,872 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@452f3c19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121028cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129ea183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c39c3d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43136ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13698df1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,880 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf4c816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0c300f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:06,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:06,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ad421a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:06,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a016b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:06,884 DEBUG [Thread-2193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,884 DEBUG [Thread-2195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,884 DEBUG [Thread-2182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,886 DEBUG [Thread-2197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:06,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T19:45:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,889 DEBUG [Thread-2184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:45:06,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,895 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:06,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:06,895 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,896 DEBUG [Thread-2191 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,897 DEBUG [Thread-2191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,900 DEBUG [Thread-2191 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,905 DEBUG [Thread-2186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,908 DEBUG [Thread-2186 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,908 DEBUG [Thread-2186 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,915 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:06,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:06,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,925 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,927 DEBUG [Thread-2180 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,927 DEBUG [Thread-2180 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,939 DEBUG [Thread-2188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:06,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2] 2024-11-13T19:45:06,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/95d0f54cca9b4db2b8f17cc87abead34 is 50, key is test_row_0/A:col10/1731527106912/Put/seqid=0 2024-11-13T19:45:06,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742316_1492 (size=14337) 2024-11-13T19:45:06,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/95d0f54cca9b4db2b8f17cc87abead34 2024-11-13T19:45:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:45:07,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ea3c3f4a0e7141bab712536d3368d9f8 is 50, key is test_row_0/B:col10/1731527106912/Put/seqid=0 2024-11-13T19:45:07,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527107013, max timestamp=1731527107018Num uniques 859; 2024-11-13T19:45:07,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:07,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527107013, max timestamp=1731527107018Num uniques 859; 2024-11-13T19:45:07,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:07,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527107013, max timestamp=1731527107018Num uniques 859; 2024-11-13T19:45:07,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:07,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742317_1493 (size=9657) 2024-11-13T19:45:07,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ea3c3f4a0e7141bab712536d3368d9f8 2024-11-13T19:45:07,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T19:45:07,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:07,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0dc0ce0c01e54e858f1ca61f5d63f4ab is 50, key is test_row_0/C:col10/1731527106912/Put/seqid=0 2024-11-13T19:45:07,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742318_1494 (size=9657) 2024-11-13T19:45:07,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527167077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527167078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527167080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527167081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527167084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T19:45:07,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:07,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527167204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527167204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527167204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527167204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527167204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T19:45:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527167424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527167424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527167425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527167425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527167425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0dc0ce0c01e54e858f1ca61f5d63f4ab 2024-11-13T19:45:07,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/95d0f54cca9b4db2b8f17cc87abead34 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34 2024-11-13T19:45:07,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34, entries=200, sequenceid=11, filesize=14.0 K 2024-11-13T19:45:07,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ea3c3f4a0e7141bab712536d3368d9f8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8 2024-11-13T19:45:07,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8, entries=100, sequenceid=11, filesize=9.4 K 2024-11-13T19:45:07,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0dc0ce0c01e54e858f1ca61f5d63f4ab as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab 2024-11-13T19:45:07,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab, entries=100, sequenceid=11, filesize=9.4 K 2024-11-13T19:45:07,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e12f682899aa55a4d182e84dc252f667 in 596ms, sequenceid=11, compaction requested=false 2024-11-13T19:45:07,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:07,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:07,514 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:07,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:07,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:07,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/0c7803b99f0c41e991be1a3663c5eb3f is 50, key is test_row_0/A:col10/1731527107074/Put/seqid=0 2024-11-13T19:45:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:45:07,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742319_1495 (size=12001) 2024-11-13T19:45:07,534 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/0c7803b99f0c41e991be1a3663c5eb3f 2024-11-13T19:45:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/22c8002175464b08bce1ce2aeb11ebf4 is 50, key is test_row_0/B:col10/1731527107074/Put/seqid=0 2024-11-13T19:45:07,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742320_1496 (size=12001) 2024-11-13T19:45:07,555 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/22c8002175464b08bce1ce2aeb11ebf4 2024-11-13T19:45:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/18492c94564540169447f1b66e0851bb is 50, key is test_row_0/C:col10/1731527107074/Put/seqid=0 2024-11-13T19:45:07,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742321_1497 (size=12001) 2024-11-13T19:45:07,707 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:45:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:07,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527167753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527167753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527167754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527167755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527167772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527167874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527167874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527167883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527167883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527167894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:07,978 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/18492c94564540169447f1b66e0851bb 2024-11-13T19:45:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/0c7803b99f0c41e991be1a3663c5eb3f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f 2024-11-13T19:45:07,988 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T19:45:07,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/22c8002175464b08bce1ce2aeb11ebf4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4 2024-11-13T19:45:07,997 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T19:45:07,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/18492c94564540169447f1b66e0851bb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb 2024-11-13T19:45:08,004 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T19:45:08,005 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e12f682899aa55a4d182e84dc252f667 in 491ms, sequenceid=48, compaction requested=false 2024-11-13T19:45:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T19:45:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T19:45:08,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T19:45:08,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-13T19:45:08,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1240 sec 2024-11-13T19:45:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:45:08,035 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:08,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T19:45:08,038 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:08,039 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:08,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:08,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:08,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:08,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b00db1ec84e445c49100e032def5398a is 50, key is test_row_0/A:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742322_1498 (size=14341) 2024-11-13T19:45:08,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b00db1ec84e445c49100e032def5398a 2024-11-13T19:45:08,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/903ea621aa294569b0b6f9fa69129082 is 50, key is test_row_0/B:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742323_1499 (size=12001) 2024-11-13T19:45:08,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/903ea621aa294569b0b6f9fa69129082 2024-11-13T19:45:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:08,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/492994672d834b5ab496fea6cc940160 is 50, key is test_row_0/C:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527108132, max timestamp=1731527108158Num uniques 859; 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527108132, max timestamp=1731527108158Num uniques 859; 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731527108132, max timestamp=1731527108158Num uniques 859; 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:45:08,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T19:45:08,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742324_1500 (size=12001) 2024-11-13T19:45:08,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/492994672d834b5ab496fea6cc940160 2024-11-13T19:45:08,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b00db1ec84e445c49100e032def5398a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a 2024-11-13T19:45:08,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a, entries=200, sequenceid=59, filesize=14.0 K 2024-11-13T19:45:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/903ea621aa294569b0b6f9fa69129082 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082 2024-11-13T19:45:08,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T19:45:08,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/492994672d834b5ab496fea6cc940160 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160 2024-11-13T19:45:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527168188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T19:45:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e12f682899aa55a4d182e84dc252f667 in 111ms, sequenceid=59, compaction requested=true 2024-11-13T19:45:08,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:08,211 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:08,211 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:08,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:08,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-13T19:45:08,212 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:08,212 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:08,212 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:08,212 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,212 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:08,212 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,212 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.7 K 2024-11-13T19:45:08,212 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=32.9 K 2024-11-13T19:45:08,213 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d0f54cca9b4db2b8f17cc87abead34, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731527106900 2024-11-13T19:45:08,213 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3c3f4a0e7141bab712536d3368d9f8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731527106912 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:08,213 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c8002175464b08bce1ce2aeb11ebf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731527107074 2024-11-13T19:45:08,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7803b99f0c41e991be1a3663c5eb3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731527107074 2024-11-13T19:45:08,214 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 903ea621aa294569b0b6f9fa69129082, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107753 2024-11-13T19:45:08,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00db1ec84e445c49100e032def5398a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107750 2024-11-13T19:45:08,223 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:08,224 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/32790ab765dd4aa2aec3b83b2cc3eea9 is 50, key is test_row_0/A:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,226 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#419 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:08,226 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/42e5820665e34af6a0cf4b00749ddbf3 is 50, key is test_row_0/B:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/d7cf5f06c6a343859ff62115e5d08d35 is 50, key is test_row_0/A:col10/1731527108202/Put/seqid=0 2024-11-13T19:45:08,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527168235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527168237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742326_1502 (size=12104) 2024-11-13T19:45:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,264 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/42e5820665e34af6a0cf4b00749ddbf3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/42e5820665e34af6a0cf4b00749ddbf3 2024-11-13T19:45:08,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742325_1501 (size=12104) 2024-11-13T19:45:08,271 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 42e5820665e34af6a0cf4b00749ddbf3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:08,271 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,271 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527108211; duration=0sec 2024-11-13T19:45:08,271 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:08,271 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:08,271 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:08,273 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/32790ab765dd4aa2aec3b83b2cc3eea9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/32790ab765dd4aa2aec3b83b2cc3eea9 2024-11-13T19:45:08,274 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:08,274 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:08,274 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,274 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=32.9 K 2024-11-13T19:45:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527168253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,275 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc0ce0c01e54e858f1ca61f5d63f4ab, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731527106912 2024-11-13T19:45:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527168254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,277 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 18492c94564540169447f1b66e0851bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731527107074 2024-11-13T19:45:08,278 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 492994672d834b5ab496fea6cc940160, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107753 2024-11-13T19:45:08,278 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 32790ab765dd4aa2aec3b83b2cc3eea9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:08,278 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,278 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527108210; duration=0sec 2024-11-13T19:45:08,279 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:08,279 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:08,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742327_1503 (size=21365) 2024-11-13T19:45:08,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/d7cf5f06c6a343859ff62115e5d08d35 2024-11-13T19:45:08,296 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:08,296 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7d24550f581349b290ae788318d4da06 is 50, key is test_row_0/C:col10/1731527107768/Put/seqid=0 2024-11-13T19:45:08,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/804e8ca440b443f3bd5b256074dbab89 is 50, key is test_row_0/B:col10/1731527108202/Put/seqid=0 2024-11-13T19:45:08,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742328_1504 (size=12104) 2024-11-13T19:45:08,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742329_1505 (size=12001) 2024-11-13T19:45:08,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/804e8ca440b443f3bd5b256074dbab89 2024-11-13T19:45:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,329 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7d24550f581349b290ae788318d4da06 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7d24550f581349b290ae788318d4da06 2024-11-13T19:45:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527168314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f959025c88604cf4ac52cd0c467d43e0 is 50, key is test_row_0/C:col10/1731527108202/Put/seqid=0 2024-11-13T19:45:08,336 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 7d24550f581349b290ae788318d4da06(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:08,336 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,336 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527108211; duration=0sec 2024-11-13T19:45:08,336 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:08,337 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:08,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742330_1506 (size=12001) 2024-11-13T19:45:08,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f959025c88604cf4ac52cd0c467d43e0 2024-11-13T19:45:08,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/d7cf5f06c6a343859ff62115e5d08d35 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35 2024-11-13T19:45:08,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35, entries=350, sequenceid=94, filesize=20.9 K 2024-11-13T19:45:08,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/804e8ca440b443f3bd5b256074dbab89 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89 2024-11-13T19:45:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527168365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527168365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:45:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f959025c88604cf4ac52cd0c467d43e0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0 2024-11-13T19:45:08,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:45:08,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for e12f682899aa55a4d182e84dc252f667 in 179ms, sequenceid=94, compaction requested=false 2024-11-13T19:45:08,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:08,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:45:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:08,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:08,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5b0c21fd60a444da8df45c23dc1b32f1 is 50, key is test_row_0/A:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:08,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742331_1507 (size=16681) 2024-11-13T19:45:08,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5b0c21fd60a444da8df45c23dc1b32f1 2024-11-13T19:45:08,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/dd6d92edc33e4d559d6693a02e5013e0 is 50, key is test_row_0/B:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:08,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742332_1508 (size=12001) 2024-11-13T19:45:08,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527108492, max timestamp=1731527108528Num uniques 859; 2024-11-13T19:45:08,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:08,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527108492, max timestamp=1731527108528Num uniques 859; 2024-11-13T19:45:08,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:08,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527108492, max timestamp=1731527108528Num uniques 859; 2024-11-13T19:45:08,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:08,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527168597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527168599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527168607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527168611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527168612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527168744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/dd6d92edc33e4d559d6693a02e5013e0 2024-11-13T19:45:08,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/66a124f122af481cab45f31afe0ee7d9 is 50, key is test_row_0/C:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742333_1509 (size=12001) 2024-11-13T19:45:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:08,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:08,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:08,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:08,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527168964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:09,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527169264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527169264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527169274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527169274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:09,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:09,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:09,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/66a124f122af481cab45f31afe0ee7d9 2024-11-13T19:45:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527169283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5b0c21fd60a444da8df45c23dc1b32f1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1 2024-11-13T19:45:09,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1, entries=250, sequenceid=108, filesize=16.3 K 2024-11-13T19:45:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/dd6d92edc33e4d559d6693a02e5013e0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0 2024-11-13T19:45:09,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T19:45:09,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/66a124f122af481cab45f31afe0ee7d9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9 2024-11-13T19:45:09,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9, entries=150, sequenceid=108, filesize=11.7 K 2024-11-13T19:45:09,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e12f682899aa55a4d182e84dc252f667 in 914ms, sequenceid=108, compaction requested=true 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:09,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:09,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:09,308 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:09,310 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:09,310 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:09,310 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,310 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/42e5820665e34af6a0cf4b00749ddbf3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.3 K 2024-11-13T19:45:09,310 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50150 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:09,310 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:09,310 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,310 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/32790ab765dd4aa2aec3b83b2cc3eea9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=49.0 K 2024-11-13T19:45:09,311 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32790ab765dd4aa2aec3b83b2cc3eea9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107753 2024-11-13T19:45:09,311 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 42e5820665e34af6a0cf4b00749ddbf3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107753 2024-11-13T19:45:09,311 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7cf5f06c6a343859ff62115e5d08d35, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731527108186 2024-11-13T19:45:09,311 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 804e8ca440b443f3bd5b256074dbab89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731527108202 2024-11-13T19:45:09,312 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6d92edc33e4d559d6693a02e5013e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108250 2024-11-13T19:45:09,312 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0c21fd60a444da8df45c23dc1b32f1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108233 2024-11-13T19:45:09,320 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:09,320 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:09,321 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b32fa9db55414f02842e303a1ffc784f is 50, key is test_row_0/B:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:09,321 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/3201d22ff2fa453b93794418e9a9aa3d is 50, key is test_row_0/A:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:09,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742334_1510 (size=12207) 2024-11-13T19:45:09,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742335_1511 (size=12207) 2024-11-13T19:45:09,347 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b32fa9db55414f02842e303a1ffc784f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b32fa9db55414f02842e303a1ffc784f 2024-11-13T19:45:09,355 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/3201d22ff2fa453b93794418e9a9aa3d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3201d22ff2fa453b93794418e9a9aa3d 2024-11-13T19:45:09,355 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into b32fa9db55414f02842e303a1ffc784f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:09,355 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:09,355 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527109308; duration=0sec 2024-11-13T19:45:09,356 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:09,356 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:09,356 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:09,357 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:09,357 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:09,357 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,357 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7d24550f581349b290ae788318d4da06, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.3 K 2024-11-13T19:45:09,358 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d24550f581349b290ae788318d4da06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731527107753 2024-11-13T19:45:09,359 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f959025c88604cf4ac52cd0c467d43e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731527108202 2024-11-13T19:45:09,360 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a124f122af481cab45f31afe0ee7d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108250 2024-11-13T19:45:09,361 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 3201d22ff2fa453b93794418e9a9aa3d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:09,361 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:09,361 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527109308; duration=0sec 2024-11-13T19:45:09,361 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:09,361 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:09,367 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:09,368 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/04c8620346984c5bbba47a287942f779 is 50, key is test_row_0/C:col10/1731527108394/Put/seqid=0 2024-11-13T19:45:09,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742336_1512 (size=12207) 2024-11-13T19:45:09,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T19:45:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,438 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:09,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/fbda8c3b668147d9b43f486eb9fb01b8 is 50, key is test_row_0/A:col10/1731527108596/Put/seqid=0 2024-11-13T19:45:09,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742337_1513 (size=12151) 2024-11-13T19:45:09,464 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/fbda8c3b668147d9b43f486eb9fb01b8 2024-11-13T19:45:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c4c86b4912724bf4b2ca3c3b302dae09 is 50, key is test_row_0/B:col10/1731527108596/Put/seqid=0 2024-11-13T19:45:09,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742338_1514 (size=12151) 2024-11-13T19:45:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:09,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:09,795 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/04c8620346984c5bbba47a287942f779 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04c8620346984c5bbba47a287942f779 2024-11-13T19:45:09,802 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 04c8620346984c5bbba47a287942f779(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:09,802 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:09,802 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527109308; duration=0sec 2024-11-13T19:45:09,802 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:09,802 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:09,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527169811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527169812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527169812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527169813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527169824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,841 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:45:09,898 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c4c86b4912724bf4b2ca3c3b302dae09 2024-11-13T19:45:09,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/ad07263299d145aba6896ee248c3ed66 is 50, key is test_row_0/C:col10/1731527108596/Put/seqid=0 2024-11-13T19:45:09,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742339_1515 (size=12151) 2024-11-13T19:45:09,937 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/ad07263299d145aba6896ee248c3ed66 2024-11-13T19:45:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527169934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527169934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527169934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527169937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527169943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/fbda8c3b668147d9b43f486eb9fb01b8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8 2024-11-13T19:45:09,958 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8, entries=150, sequenceid=146, filesize=11.9 K 2024-11-13T19:45:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c4c86b4912724bf4b2ca3c3b302dae09 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09 2024-11-13T19:45:09,962 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09, entries=150, sequenceid=146, filesize=11.9 K 2024-11-13T19:45:09,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/ad07263299d145aba6896ee248c3ed66 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66 2024-11-13T19:45:09,969 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66, entries=150, sequenceid=146, filesize=11.9 K 2024-11-13T19:45:09,970 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e12f682899aa55a4d182e84dc252f667 in 532ms, sequenceid=146, compaction requested=false 2024-11-13T19:45:09,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:09,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:09,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T19:45:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T19:45:09,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T19:45:09,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-11-13T19:45:09,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9400 sec 2024-11-13T19:45:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:10,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:45:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:10,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/102e14987dcd4749bbff14812edce779 is 50, key is test_row_0/A:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:45:10,188 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:10,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-13T19:45:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:10,195 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:10,195 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:10,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:10,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742340_1516 (size=12151) 2024-11-13T19:45:10,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/102e14987dcd4749bbff14812edce779 2024-11-13T19:45:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527170238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527170240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527170241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527170244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527170257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/84478d7126ba40f087029c159cb5edbc is 50, key is test_row_0/B:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:10,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742341_1517 (size=12151) 2024-11-13T19:45:10,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/84478d7126ba40f087029c159cb5edbc 2024-11-13T19:45:10,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/c2e5cd3e127e4b8891f2fdff5d87e58a is 50, key is test_row_0/C:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:10,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T19:45:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742342_1518 (size=12151) 2024-11-13T19:45:10,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527170377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527170377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527170377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527170378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527170383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:10,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T19:45:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527170593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527170593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527170604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527170604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527170605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:10,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T19:45:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:10,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/c2e5cd3e127e4b8891f2fdff5d87e58a 2024-11-13T19:45:10,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/102e14987dcd4749bbff14812edce779 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779 2024-11-13T19:45:10,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T19:45:10,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/84478d7126ba40f087029c159cb5edbc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc 2024-11-13T19:45:10,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T19:45:10,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/c2e5cd3e127e4b8891f2fdff5d87e58a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a 2024-11-13T19:45:10,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T19:45:10,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e12f682899aa55a4d182e84dc252f667 in 641ms, sequenceid=161, compaction requested=true 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:10,807 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:10,807 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:10,808 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:10,808 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:10,809 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,809 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b32fa9db55414f02842e303a1ffc784f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.7 K 2024-11-13T19:45:10,809 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:10,809 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:10,809 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,809 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3201d22ff2fa453b93794418e9a9aa3d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.7 K 2024-11-13T19:45:10,810 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b32fa9db55414f02842e303a1ffc784f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108250 2024-11-13T19:45:10,810 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3201d22ff2fa453b93794418e9a9aa3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108250 2024-11-13T19:45:10,810 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c86b4912724bf4b2ca3c3b302dae09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731527108596 2024-11-13T19:45:10,810 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbda8c3b668147d9b43f486eb9fb01b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731527108596 2024-11-13T19:45:10,811 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102e14987dcd4749bbff14812edce779, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:10,811 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 84478d7126ba40f087029c159cb5edbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:10,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,825 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:10,837 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:10,837 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/fa592e94c3a94fc688730e6b1aabe87e is 50, key is test_row_0/B:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:10,843 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:10,844 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4cc0395f3563496abaf4305b0405ae8a is 50, key is test_row_0/A:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1b11cfd972e046de9fa8b9990edcedd5 is 50, key is test_row_0/A:col10/1731527110254/Put/seqid=0 2024-11-13T19:45:10,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742343_1519 (size=12459) 2024-11-13T19:45:10,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:10,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742344_1520 (size=12459) 2024-11-13T19:45:10,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742345_1521 (size=12151) 2024-11-13T19:45:10,930 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1b11cfd972e046de9fa8b9990edcedd5 2024-11-13T19:45:10,936 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4cc0395f3563496abaf4305b0405ae8a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4cc0395f3563496abaf4305b0405ae8a 2024-11-13T19:45:10,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/faefb5ea99d9446a87ec1c463effd6e2 is 50, key is test_row_0/B:col10/1731527110254/Put/seqid=0 2024-11-13T19:45:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742346_1522 (size=12151) 2024-11-13T19:45:10,988 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 4cc0395f3563496abaf4305b0405ae8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:10,988 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:10,988 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527110807; duration=0sec 2024-11-13T19:45:10,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:10,989 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:10,990 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:10,996 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:10,996 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:10,996 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:10,996 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04c8620346984c5bbba47a287942f779, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.7 K 2024-11-13T19:45:10,997 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04c8620346984c5bbba47a287942f779, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731527108250 2024-11-13T19:45:10,997 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad07263299d145aba6896ee248c3ed66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731527108596 2024-11-13T19:45:10,998 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e5cd3e127e4b8891f2fdff5d87e58a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:11,009 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:11,009 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4dae4e6c4aa5428fbf20b06b56798be4 is 50, key is test_row_0/C:col10/1731527110163/Put/seqid=0 2024-11-13T19:45:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527170990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527171010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527171010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527171011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527171012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742347_1523 (size=12459) 2024-11-13T19:45:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527171123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527171145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527171145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527171145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527171145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,323 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/fa592e94c3a94fc688730e6b1aabe87e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fa592e94c3a94fc688730e6b1aabe87e 2024-11-13T19:45:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:11,340 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into fa592e94c3a94fc688730e6b1aabe87e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:11,340 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:11,340 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527110807; duration=0sec 2024-11-13T19:45:11,340 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:11,340 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:11,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527171343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527171354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527171354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527171354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527171354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,373 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/faefb5ea99d9446a87ec1c463effd6e2 2024-11-13T19:45:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7cd01d5efd9641be98188f176fe6b12c is 50, key is test_row_0/C:col10/1731527110254/Put/seqid=0 2024-11-13T19:45:11,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742348_1524 (size=12151) 2024-11-13T19:45:11,400 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7cd01d5efd9641be98188f176fe6b12c 2024-11-13T19:45:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1b11cfd972e046de9fa8b9990edcedd5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5 2024-11-13T19:45:11,408 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T19:45:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/faefb5ea99d9446a87ec1c463effd6e2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2 2024-11-13T19:45:11,412 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T19:45:11,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7cd01d5efd9641be98188f176fe6b12c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c 2024-11-13T19:45:11,419 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T19:45:11,420 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e12f682899aa55a4d182e84dc252f667 in 595ms, sequenceid=179, compaction requested=false 2024-11-13T19:45:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-13T19:45:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-13T19:45:11,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T19:45:11,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2250 sec 2024-11-13T19:45:11,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.2360 sec 2024-11-13T19:45:11,483 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4dae4e6c4aa5428fbf20b06b56798be4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4dae4e6c4aa5428fbf20b06b56798be4 2024-11-13T19:45:11,491 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 4dae4e6c4aa5428fbf20b06b56798be4(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:11,491 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:11,491 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527110807; duration=0sec 2024-11-13T19:45:11,491 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:11,491 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:11,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:11,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:11,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/6b32608219c8418f9a95e91fa437a4b0 is 50, key is test_row_0/A:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:11,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742349_1525 (size=14541) 2024-11-13T19:45:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527171702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527171704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527171713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527171715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527171715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527171824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527171824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527171824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527171833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527171833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527172043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527172043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527172044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527172054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527172054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/6b32608219c8418f9a95e91fa437a4b0 2024-11-13T19:45:12,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/551e08ec51fa428f99d17bc4598ae06d is 50, key is test_row_0/B:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:12,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742350_1526 (size=12151) 2024-11-13T19:45:12,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/551e08ec51fa428f99d17bc4598ae06d 2024-11-13T19:45:12,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/5019439737a249d79d7cbfcabe1829c5 is 50, key is test_row_0/C:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742351_1527 (size=12151) 2024-11-13T19:45:12,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/5019439737a249d79d7cbfcabe1829c5 2024-11-13T19:45:12,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/6b32608219c8418f9a95e91fa437a4b0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0 2024-11-13T19:45:12,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0, entries=200, sequenceid=197, filesize=14.2 K 2024-11-13T19:45:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/551e08ec51fa428f99d17bc4598ae06d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d 2024-11-13T19:45:12,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T19:45:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/5019439737a249d79d7cbfcabe1829c5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5 2024-11-13T19:45:12,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T19:45:12,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e12f682899aa55a4d182e84dc252f667 in 479ms, sequenceid=197, compaction requested=true 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:12,148 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:45:12,148 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:12,150 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:12,150 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:12,150 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,150 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fa592e94c3a94fc688730e6b1aabe87e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.9 K 2024-11-13T19:45:12,150 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:12,150 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:12,150 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,150 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4cc0395f3563496abaf4305b0405ae8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=38.2 K 2024-11-13T19:45:12,151 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting fa592e94c3a94fc688730e6b1aabe87e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:12,151 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cc0395f3563496abaf4305b0405ae8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:12,153 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b11cfd972e046de9fa8b9990edcedd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731527110239 2024-11-13T19:45:12,153 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting faefb5ea99d9446a87ec1c463effd6e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731527110239 2024-11-13T19:45:12,154 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b32608219c8418f9a95e91fa437a4b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110941 2024-11-13T19:45:12,155 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 551e08ec51fa428f99d17bc4598ae06d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110982 2024-11-13T19:45:12,161 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:12,161 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/21a7ab68afb34ef2b268a715b2bc42fa is 50, key is test_row_0/A:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:12,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742352_1528 (size=12561) 2024-11-13T19:45:12,178 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:12,179 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/64962c83b6c24b309e72afffbba4fb2e is 50, key is test_row_0/B:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742353_1529 (size=12561) 2024-11-13T19:45:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T19:45:12,344 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:12,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-13T19:45:12,351 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:12,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:45:12,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:12,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5170eb2fa7c04cc6bbfe02f11ce69335 is 50, key is test_row_0/A:col10/1731527111711/Put/seqid=0 2024-11-13T19:45:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742354_1530 (size=14541) 2024-11-13T19:45:12,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527172408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527172411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527172412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527172415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527172417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:45:12,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T19:45:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527172534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527172544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527172544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527172544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527172544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,575 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/21a7ab68afb34ef2b268a715b2bc42fa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/21a7ab68afb34ef2b268a715b2bc42fa 2024-11-13T19:45:12,586 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 21a7ab68afb34ef2b268a715b2bc42fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:12,587 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:12,587 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527112148; duration=0sec 2024-11-13T19:45:12,587 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:12,587 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:12,587 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:12,606 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:12,606 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:12,606 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,607 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4dae4e6c4aa5428fbf20b06b56798be4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=35.9 K 2024-11-13T19:45:12,609 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dae4e6c4aa5428fbf20b06b56798be4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527109807 2024-11-13T19:45:12,614 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd01d5efd9641be98188f176fe6b12c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731527110239 2024-11-13T19:45:12,616 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5019439737a249d79d7cbfcabe1829c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110982 2024-11-13T19:45:12,617 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/64962c83b6c24b309e72afffbba4fb2e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/64962c83b6c24b309e72afffbba4fb2e 2024-11-13T19:45:12,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T19:45:12,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,665 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 64962c83b6c24b309e72afffbba4fb2e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:12,665 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:12,665 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527112148; duration=0sec 2024-11-13T19:45:12,665 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:12,665 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:45:12,694 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:45:12,695 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/fd9e797497cc44449ebc50d82999d143 is 50, key is test_row_0/C:col10/1731527111010/Put/seqid=0 2024-11-13T19:45:12,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742355_1531 (size=12561) 2024-11-13T19:45:12,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527172754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527172765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527172765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527172765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527172765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:12,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5170eb2fa7c04cc6bbfe02f11ce69335 2024-11-13T19:45:12,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T19:45:12,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2cee5e749c8844f99be61c3f8f3ac44d is 50, key is test_row_0/B:col10/1731527111711/Put/seqid=0 2024-11-13T19:45:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:12,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742356_1532 (size=12151) 2024-11-13T19:45:12,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2cee5e749c8844f99be61c3f8f3ac44d 2024-11-13T19:45:12,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/05918dcc92f046b0b638f4434b840d6c is 50, key is test_row_0/C:col10/1731527111711/Put/seqid=0 2024-11-13T19:45:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742357_1533 (size=12151) 2024-11-13T19:45:12,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/05918dcc92f046b0b638f4434b840d6c 2024-11-13T19:45:12,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5170eb2fa7c04cc6bbfe02f11ce69335 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335 2024-11-13T19:45:12,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335, entries=200, sequenceid=216, filesize=14.2 K 2024-11-13T19:45:12,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2cee5e749c8844f99be61c3f8f3ac44d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d 2024-11-13T19:45:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T19:45:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/05918dcc92f046b0b638f4434b840d6c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c 2024-11-13T19:45:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T19:45:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e12f682899aa55a4d182e84dc252f667 in 511ms, sequenceid=216, compaction requested=false 2024-11-13T19:45:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:12,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T19:45:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:12,977 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1fe786091096457993a8a8201566a571 is 50, key is test_row_0/A:col10/1731527112412/Put/seqid=0 2024-11-13T19:45:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:45:12,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742358_1534 (size=12151) 2024-11-13T19:45:12,997 DEBUG [regionserver/54d0784d1943:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-13T19:45:13,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/32790ab765dd4aa2aec3b83b2cc3eea9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3201d22ff2fa453b93794418e9a9aa3d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4cc0395f3563496abaf4305b0405ae8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0] to archive 2024-11-13T19:45:13,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:13,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/95d0f54cca9b4db2b8f17cc87abead34 2024-11-13T19:45:13,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/0c7803b99f0c41e991be1a3663c5eb3f 2024-11-13T19:45:13,005 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1fe786091096457993a8a8201566a571 2024-11-13T19:45:13,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b00db1ec84e445c49100e032def5398a 2024-11-13T19:45:13,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/32790ab765dd4aa2aec3b83b2cc3eea9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/32790ab765dd4aa2aec3b83b2cc3eea9 2024-11-13T19:45:13,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/d7cf5f06c6a343859ff62115e5d08d35 2024-11-13T19:45:13,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5b0c21fd60a444da8df45c23dc1b32f1 2024-11-13T19:45:13,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3201d22ff2fa453b93794418e9a9aa3d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3201d22ff2fa453b93794418e9a9aa3d 2024-11-13T19:45:13,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/fbda8c3b668147d9b43f486eb9fb01b8 2024-11-13T19:45:13,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4cc0395f3563496abaf4305b0405ae8a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4cc0395f3563496abaf4305b0405ae8a 2024-11-13T19:45:13,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1b931b1d0e3142e7bdd57286354ca0d3 is 50, key is test_row_0/B:col10/1731527112412/Put/seqid=0 2024-11-13T19:45:13,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/102e14987dcd4749bbff14812edce779 2024-11-13T19:45:13,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1b11cfd972e046de9fa8b9990edcedd5 2024-11-13T19:45:13,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/6b32608219c8418f9a95e91fa437a4b0 2024-11-13T19:45:13,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/42e5820665e34af6a0cf4b00749ddbf3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b32fa9db55414f02842e303a1ffc784f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fa592e94c3a94fc688730e6b1aabe87e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d] to archive 2024-11-13T19:45:13,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:13,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ea3c3f4a0e7141bab712536d3368d9f8 2024-11-13T19:45:13,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/22c8002175464b08bce1ce2aeb11ebf4 2024-11-13T19:45:13,049 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/42e5820665e34af6a0cf4b00749ddbf3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/42e5820665e34af6a0cf4b00749ddbf3 2024-11-13T19:45:13,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/903ea621aa294569b0b6f9fa69129082 2024-11-13T19:45:13,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/804e8ca440b443f3bd5b256074dbab89 2024-11-13T19:45:13,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b32fa9db55414f02842e303a1ffc784f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b32fa9db55414f02842e303a1ffc784f 2024-11-13T19:45:13,058 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/dd6d92edc33e4d559d6693a02e5013e0 2024-11-13T19:45:13,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c4c86b4912724bf4b2ca3c3b302dae09 2024-11-13T19:45:13,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fa592e94c3a94fc688730e6b1aabe87e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fa592e94c3a94fc688730e6b1aabe87e 2024-11-13T19:45:13,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/84478d7126ba40f087029c159cb5edbc 2024-11-13T19:45:13,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/faefb5ea99d9446a87ec1c463effd6e2 2024-11-13T19:45:13,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/551e08ec51fa428f99d17bc4598ae06d 2024-11-13T19:45:13,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7d24550f581349b290ae788318d4da06, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04c8620346984c5bbba47a287942f779, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a] to archive 2024-11-13T19:45:13,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:13,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0dc0ce0c01e54e858f1ca61f5d63f4ab 2024-11-13T19:45:13,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/18492c94564540169447f1b66e0851bb 2024-11-13T19:45:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:13,085 WARN [regionserver/54d0784d1943:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-13T19:45:13,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7d24550f581349b290ae788318d4da06 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7d24550f581349b290ae788318d4da06 2024-11-13T19:45:13,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742359_1535 (size=12151) 2024-11-13T19:45:13,087 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1b931b1d0e3142e7bdd57286354ca0d3 2024-11-13T19:45:13,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/492994672d834b5ab496fea6cc940160 2024-11-13T19:45:13,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f959025c88604cf4ac52cd0c467d43e0 2024-11-13T19:45:13,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04c8620346984c5bbba47a287942f779 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04c8620346984c5bbba47a287942f779 2024-11-13T19:45:13,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/66a124f122af481cab45f31afe0ee7d9 2024-11-13T19:45:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/8469bffa8f6846dab75c75b207d81bd5 is 50, key is test_row_0/C:col10/1731527112412/Put/seqid=0 2024-11-13T19:45:13,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/ad07263299d145aba6896ee248c3ed66 2024-11-13T19:45:13,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/c2e5cd3e127e4b8891f2fdff5d87e58a 2024-11-13T19:45:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742360_1536 (size=12151) 2024-11-13T19:45:13,115 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/8469bffa8f6846dab75c75b207d81bd5 2024-11-13T19:45:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1fe786091096457993a8a8201566a571 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571 2024-11-13T19:45:13,126 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/fd9e797497cc44449ebc50d82999d143 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/fd9e797497cc44449ebc50d82999d143 2024-11-13T19:45:13,127 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571, entries=150, sequenceid=231, filesize=11.9 K 2024-11-13T19:45:13,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1b931b1d0e3142e7bdd57286354ca0d3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3 2024-11-13T19:45:13,132 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into fd9e797497cc44449ebc50d82999d143(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:13,132 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,132 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527112148; duration=0sec 2024-11-13T19:45:13,132 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:13,132 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:13,133 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3, entries=150, sequenceid=231, filesize=11.9 K 2024-11-13T19:45:13,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/8469bffa8f6846dab75c75b207d81bd5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5 2024-11-13T19:45:13,141 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-13T19:45:13,144 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e12f682899aa55a4d182e84dc252f667 in 167ms, sequenceid=231, compaction requested=true 2024-11-13T19:45:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-13T19:45:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-13T19:45:13,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T19:45:13,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 791 msec 2024-11-13T19:45:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:13,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:45:13,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:13,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 806 msec 2024-11-13T19:45:13,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/bcded46d2ef14e0ba66f0d69f9d3b82e is 50, key is test_row_0/A:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742361_1537 (size=14541) 2024-11-13T19:45:13,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/bcded46d2ef14e0ba66f0d69f9d3b82e 2024-11-13T19:45:13,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1f518946665049a197a8fb3e488b6e96 is 50, key is test_row_0/B:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527173167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527173169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527173178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527173180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527173181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742362_1538 (size=12151) 2024-11-13T19:45:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1f518946665049a197a8fb3e488b6e96 2024-11-13T19:45:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/372f09fac8b0425ebebf381bc4404ec1 is 50, key is test_row_0/C:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742363_1539 (size=12151) 2024-11-13T19:45:13,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/372f09fac8b0425ebebf381bc4404ec1 2024-11-13T19:45:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/bcded46d2ef14e0ba66f0d69f9d3b82e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e 2024-11-13T19:45:13,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e, entries=200, sequenceid=252, filesize=14.2 K 2024-11-13T19:45:13,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1f518946665049a197a8fb3e488b6e96 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96 2024-11-13T19:45:13,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:45:13,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/372f09fac8b0425ebebf381bc4404ec1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1 2024-11-13T19:45:13,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:45:13,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e12f682899aa55a4d182e84dc252f667 in 102ms, sequenceid=252, compaction requested=true 2024-11-13T19:45:13,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:13,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:13,253 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:13,253 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:13,255 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:13,255 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:13,255 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,255 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/21a7ab68afb34ef2b268a715b2bc42fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=52.5 K 2024-11-13T19:45:13,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:13,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:13,256 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,256 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/64962c83b6c24b309e72afffbba4fb2e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=47.9 K 2024-11-13T19:45:13,256 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a7ab68afb34ef2b268a715b2bc42fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110982 2024-11-13T19:45:13,256 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 64962c83b6c24b309e72afffbba4fb2e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110982 2024-11-13T19:45:13,257 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5170eb2fa7c04cc6bbfe02f11ce69335, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731527111711 2024-11-13T19:45:13,257 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cee5e749c8844f99be61c3f8f3ac44d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731527111711 2024-11-13T19:45:13,257 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe786091096457993a8a8201566a571, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527112409 2024-11-13T19:45:13,257 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b931b1d0e3142e7bdd57286354ca0d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527112409 2024-11-13T19:45:13,257 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcded46d2ef14e0ba66f0d69f9d3b82e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:13,258 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f518946665049a197a8fb3e488b6e96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:13,272 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:13,273 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/db8945db5a214303b697f66fecdcfa84 is 50, key is test_row_0/A:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,276 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:13,276 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/71b2dd9a445e4b81b857c82369b49f5e is 50, key is test_row_0/B:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:13,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:13,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742364_1540 (size=12289) 2024-11-13T19:45:13,314 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/db8945db5a214303b697f66fecdcfa84 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db8945db5a214303b697f66fecdcfa84 2024-11-13T19:45:13,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/97e584829f404209b10c5a6dfb5da4e3 is 50, key is test_row_0/A:col10/1731527113289/Put/seqid=0 2024-11-13T19:45:13,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742365_1541 (size=12289) 2024-11-13T19:45:13,326 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into db8945db5a214303b697f66fecdcfa84(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:13,326 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,326 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=12, startTime=1731527113253; duration=0sec 2024-11-13T19:45:13,326 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:13,326 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:13,326 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:13,328 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:13,328 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:13,328 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,328 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/fd9e797497cc44449ebc50d82999d143, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=47.9 K 2024-11-13T19:45:13,329 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9e797497cc44449ebc50d82999d143, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731527110982 2024-11-13T19:45:13,329 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05918dcc92f046b0b638f4434b840d6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731527111711 2024-11-13T19:45:13,329 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8469bffa8f6846dab75c75b207d81bd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527112409 2024-11-13T19:45:13,330 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 372f09fac8b0425ebebf381bc4404ec1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:13,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742366_1542 (size=14741) 2024-11-13T19:45:13,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/97e584829f404209b10c5a6dfb5da4e3 2024-11-13T19:45:13,339 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#460 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:13,339 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a4211c9d06204a27a53bdac15fb79673 is 50, key is test_row_0/C:col10/1731527113109/Put/seqid=0 2024-11-13T19:45:13,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/db06653fafbe4985a546125c9ad0cac8 is 50, key is test_row_0/B:col10/1731527113289/Put/seqid=0 2024-11-13T19:45:13,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742367_1543 (size=12391) 2024-11-13T19:45:13,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742368_1544 (size=12301) 2024-11-13T19:45:13,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/db06653fafbe4985a546125c9ad0cac8 2024-11-13T19:45:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527173354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527173358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9ea57c7f10da4bbe85a0cc72688a6169 is 50, key is test_row_0/C:col10/1731527113289/Put/seqid=0 2024-11-13T19:45:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527173371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527173373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527173375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742369_1545 (size=12301) 2024-11-13T19:45:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527173484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527173484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:45:13,494 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:13,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-13T19:45:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T19:45:13,496 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:13,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:13,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:13,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527173493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527173493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527173494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T19:45:13,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T19:45:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527173694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527173694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527173713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527173713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,725 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/71b2dd9a445e4b81b857c82369b49f5e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71b2dd9a445e4b81b857c82369b49f5e 2024-11-13T19:45:13,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527173713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:13,730 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 71b2dd9a445e4b81b857c82369b49f5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:13,730 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,730 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=12, startTime=1731527113253; duration=0sec 2024-11-13T19:45:13,730 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:13,730 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:13,769 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a4211c9d06204a27a53bdac15fb79673 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a4211c9d06204a27a53bdac15fb79673 2024-11-13T19:45:13,773 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into a4211c9d06204a27a53bdac15fb79673(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:13,773 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,773 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=12, startTime=1731527113254; duration=0sec 2024-11-13T19:45:13,773 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:13,773 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:13,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9ea57c7f10da4bbe85a0cc72688a6169 2024-11-13T19:45:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/97e584829f404209b10c5a6dfb5da4e3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3 2024-11-13T19:45:13,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T19:45:13,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:13,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:13,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3, entries=200, sequenceid=265, filesize=14.4 K 2024-11-13T19:45:13,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/db06653fafbe4985a546125c9ad0cac8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8 2024-11-13T19:45:13,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8, entries=150, sequenceid=265, filesize=12.0 K 2024-11-13T19:45:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T19:45:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9ea57c7f10da4bbe85a0cc72688a6169 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169 2024-11-13T19:45:13,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169, entries=150, sequenceid=265, filesize=12.0 K 2024-11-13T19:45:13,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e12f682899aa55a4d182e84dc252f667 in 527ms, sequenceid=265, compaction requested=false 2024-11-13T19:45:13,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:13,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:13,958 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f000770d36a34a8699a6499cec8a7b0d is 50, key is test_row_0/A:col10/1731527113367/Put/seqid=0 2024-11-13T19:45:13,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742370_1546 (size=12301) 2024-11-13T19:45:13,966 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f000770d36a34a8699a6499cec8a7b0d 2024-11-13T19:45:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/7470bd3a6e66486584b8196610f98a82 is 50, key is test_row_0/B:col10/1731527113367/Put/seqid=0 2024-11-13T19:45:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742371_1547 (size=12301) 2024-11-13T19:45:13,981 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/7470bd3a6e66486584b8196610f98a82 2024-11-13T19:45:13,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/efcec0b15abf40aeb1b2ec7e5c98ce9d is 50, key is test_row_0/C:col10/1731527113367/Put/seqid=0 2024-11-13T19:45:14,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742372_1548 (size=12301) 2024-11-13T19:45:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:14,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:14,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527174048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527174050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527174050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527174052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527174052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T19:45:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527174164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527174173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527174173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527174173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527174173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527174383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527174393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527174394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527174394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527174394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,405 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/efcec0b15abf40aeb1b2ec7e5c98ce9d 2024-11-13T19:45:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f000770d36a34a8699a6499cec8a7b0d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d 2024-11-13T19:45:14,416 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:45:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/7470bd3a6e66486584b8196610f98a82 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82 2024-11-13T19:45:14,421 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:45:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/efcec0b15abf40aeb1b2ec7e5c98ce9d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d 2024-11-13T19:45:14,425 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:45:14,426 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e12f682899aa55a4d182e84dc252f667 in 468ms, sequenceid=287, compaction requested=true 2024-11-13T19:45:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-13T19:45:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-13T19:45:14,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T19:45:14,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 931 msec 2024-11-13T19:45:14,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 937 msec 2024-11-13T19:45:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T19:45:14,634 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:14,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T19:45:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:14,637 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:14,637 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:14,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:14,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:14,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1a1e08c8249840b48ded9daed332fd69 is 50, key is test_row_0/A:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742373_1549 (size=14741) 2024-11-13T19:45:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527174750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527174752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527174762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527174774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527174774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:45:14,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:14,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:14,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:14,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527174874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527174884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527174893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527174893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527174894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:14,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:45:14,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:14,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:14,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:14,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:15,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:45:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:15,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527175093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527175104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1a1e08c8249840b48ded9daed332fd69 2024-11-13T19:45:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527175113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527175114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527175114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/e29ca054813e4ed7b398e0a24f720a1e is 50, key is test_row_0/B:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742374_1550 (size=12301) 2024-11-13T19:45:15,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/e29ca054813e4ed7b398e0a24f720a1e 2024-11-13T19:45:15,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/383b830eaac04ddca7dd7664d142f13e is 50, key is test_row_0/C:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742375_1551 (size=12301) 2024-11-13T19:45:15,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/383b830eaac04ddca7dd7664d142f13e 2024-11-13T19:45:15,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/1a1e08c8249840b48ded9daed332fd69 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69 2024-11-13T19:45:15,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69, entries=200, sequenceid=301, filesize=14.4 K 2024-11-13T19:45:15,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/e29ca054813e4ed7b398e0a24f720a1e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e 2024-11-13T19:45:15,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e, entries=150, sequenceid=301, filesize=12.0 K 2024-11-13T19:45:15,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/383b830eaac04ddca7dd7664d142f13e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e 2024-11-13T19:45:15,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e, entries=150, sequenceid=301, filesize=12.0 K 2024-11-13T19:45:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e12f682899aa55a4d182e84dc252f667 in 494ms, sequenceid=301, compaction requested=true 2024-11-13T19:45:15,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:15,206 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:15,207 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:15,210 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:15,210 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:15,210 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,210 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db8945db5a214303b697f66fecdcfa84, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=52.8 K 2024-11-13T19:45:15,211 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting db8945db5a214303b697f66fecdcfa84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:15,213 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:15,213 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:15,213 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,213 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71b2dd9a445e4b81b857c82369b49f5e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=48.0 K 2024-11-13T19:45:15,213 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e584829f404209b10c5a6dfb5da4e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731527113159 2024-11-13T19:45:15,214 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b2dd9a445e4b81b857c82369b49f5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:15,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f000770d36a34a8699a6499cec8a7b0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527113356 2024-11-13T19:45:15,214 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting db06653fafbe4985a546125c9ad0cac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731527113178 2024-11-13T19:45:15,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1e08c8249840b48ded9daed332fd69, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:15,215 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7470bd3a6e66486584b8196610f98a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527113356 2024-11-13T19:45:15,215 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e29ca054813e4ed7b398e0a24f720a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:15,228 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:15,229 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b131c45edaec4a3fb05891d171c982ee is 50, key is test_row_0/A:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:15,236 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:15,236 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/62acd14e4c5741f890584705e608dc6c is 50, key is test_row_0/B:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:15,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:45:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,261 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:15,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742376_1552 (size=12575) 2024-11-13T19:45:15,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5efa4d0b615f4663b1a8cb954ba35612 is 50, key is test_row_0/A:col10/1731527114748/Put/seqid=0 2024-11-13T19:45:15,297 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b131c45edaec4a3fb05891d171c982ee as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b131c45edaec4a3fb05891d171c982ee 2024-11-13T19:45:15,305 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into b131c45edaec4a3fb05891d171c982ee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:15,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:15,306 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=12, startTime=1731527115206; duration=0sec 2024-11-13T19:45:15,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:15,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:15,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:15,307 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:15,307 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:15,307 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,307 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a4211c9d06204a27a53bdac15fb79673, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=48.1 K 2024-11-13T19:45:15,307 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4211c9d06204a27a53bdac15fb79673, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731527113109 2024-11-13T19:45:15,308 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea57c7f10da4bbe85a0cc72688a6169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731527113178 2024-11-13T19:45:15,309 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting efcec0b15abf40aeb1b2ec7e5c98ce9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731527113356 2024-11-13T19:45:15,309 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383b830eaac04ddca7dd7664d142f13e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:15,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742377_1553 (size=12575) 2024-11-13T19:45:15,322 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/62acd14e4c5741f890584705e608dc6c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/62acd14e4c5741f890584705e608dc6c 2024-11-13T19:45:15,327 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 62acd14e4c5741f890584705e608dc6c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:15,327 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:15,327 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=12, startTime=1731527115207; duration=0sec 2024-11-13T19:45:15,327 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:15,327 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:15,337 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:15,338 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9a8d20d9c877433f94fb7668a2ac3ea6 is 50, key is test_row_0/C:col10/1731527114050/Put/seqid=0 2024-11-13T19:45:15,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742378_1554 (size=12301) 2024-11-13T19:45:15,343 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5efa4d0b615f4663b1a8cb954ba35612 2024-11-13T19:45:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742379_1555 (size=12677) 2024-11-13T19:45:15,364 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9a8d20d9c877433f94fb7668a2ac3ea6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9a8d20d9c877433f94fb7668a2ac3ea6 2024-11-13T19:45:15,370 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 9a8d20d9c877433f94fb7668a2ac3ea6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:15,370 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:15,370 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=12, startTime=1731527115207; duration=0sec 2024-11-13T19:45:15,370 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:15,370 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:15,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76a210d8f9b74066b39441c1c16006fa is 50, key is test_row_0/B:col10/1731527114748/Put/seqid=0 2024-11-13T19:45:15,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742380_1556 (size=12301) 2024-11-13T19:45:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:15,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:15,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527175466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527175467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527175469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527175482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527175482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527175594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527175595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527175597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527175597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527175597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76a210d8f9b74066b39441c1c16006fa 2024-11-13T19:45:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/73763b2318d44096833c3aeb52c8a677 is 50, key is test_row_0/C:col10/1731527114748/Put/seqid=0 2024-11-13T19:45:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527175815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527175815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527175815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527175815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527175815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:15,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742381_1557 (size=12301) 2024-11-13T19:45:15,843 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/73763b2318d44096833c3aeb52c8a677 2024-11-13T19:45:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5efa4d0b615f4663b1a8cb954ba35612 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612 2024-11-13T19:45:15,864 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:45:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76a210d8f9b74066b39441c1c16006fa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa 2024-11-13T19:45:15,869 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:45:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/73763b2318d44096833c3aeb52c8a677 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677 2024-11-13T19:45:15,877 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:45:15,880 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e12f682899aa55a4d182e84dc252f667 in 619ms, sequenceid=319, compaction requested=false 2024-11-13T19:45:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T19:45:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T19:45:15,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T19:45:15,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2430 sec 2024-11-13T19:45:15,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2500 sec 2024-11-13T19:45:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:16,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ed7af2a3936b414287ae8c5efae3e1cb is 50, key is test_row_0/A:col10/1731527115464/Put/seqid=0 2024-11-13T19:45:16,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742382_1558 (size=14741) 2024-11-13T19:45:16,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ed7af2a3936b414287ae8c5efae3e1cb 2024-11-13T19:45:16,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1e70592063fe44ed8ef47d36a5ca6736 is 50, key is test_row_0/B:col10/1731527115464/Put/seqid=0 2024-11-13T19:45:16,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527176164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527176166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527176167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527176169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742383_1559 (size=12301) 2024-11-13T19:45:16,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1e70592063fe44ed8ef47d36a5ca6736 2024-11-13T19:45:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527176177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/1c3d7be09627423db164dbccfc390c5f is 50, key is test_row_0/C:col10/1731527115464/Put/seqid=0 2024-11-13T19:45:16,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742384_1560 (size=12301) 2024-11-13T19:45:16,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/1c3d7be09627423db164dbccfc390c5f 2024-11-13T19:45:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ed7af2a3936b414287ae8c5efae3e1cb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb 2024-11-13T19:45:16,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb, entries=200, sequenceid=337, filesize=14.4 K 2024-11-13T19:45:16,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1e70592063fe44ed8ef47d36a5ca6736 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736 2024-11-13T19:45:16,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736, entries=150, sequenceid=337, filesize=12.0 K 2024-11-13T19:45:16,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/1c3d7be09627423db164dbccfc390c5f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f 2024-11-13T19:45:16,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f, entries=150, sequenceid=337, filesize=12.0 K 2024-11-13T19:45:16,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e12f682899aa55a4d182e84dc252f667 in 104ms, sequenceid=337, compaction requested=true 2024-11-13T19:45:16,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,245 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:16,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:16,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:16,245 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:16,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:16,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:16,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:16,253 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:16,253 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:16,253 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:16,253 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,253 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/62acd14e4c5741f890584705e608dc6c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.3 K 2024-11-13T19:45:16,254 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:16,254 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,254 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b131c45edaec4a3fb05891d171c982ee, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=38.7 K 2024-11-13T19:45:16,255 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 62acd14e4c5741f890584705e608dc6c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:16,255 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b131c45edaec4a3fb05891d171c982ee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:16,255 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 76a210d8f9b74066b39441c1c16006fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731527114748 2024-11-13T19:45:16,255 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5efa4d0b615f4663b1a8cb954ba35612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731527114748 2024-11-13T19:45:16,255 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e70592063fe44ed8ef47d36a5ca6736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731527115464 2024-11-13T19:45:16,256 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed7af2a3936b414287ae8c5efae3e1cb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731527115464 2024-11-13T19:45:16,267 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:16,267 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:16,267 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4b9a9b0981ea44cb85d97338530cad38 is 50, key is test_row_0/A:col10/1731527115464/Put/seqid=0 2024-11-13T19:45:16,267 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d1b97b55e1e04afe96a6d9a07848244d is 50, key is test_row_0/B:col10/1731527115464/Put/seqid=0 2024-11-13T19:45:16,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742386_1562 (size=12677) 2024-11-13T19:45:16,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742385_1561 (size=12677) 2024-11-13T19:45:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:16,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e83c4c701db848748aa7f0ef94a3d163 is 50, key is test_row_0/A:col10/1731527116167/Put/seqid=0 2024-11-13T19:45:16,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742387_1563 (size=14741) 2024-11-13T19:45:16,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e83c4c701db848748aa7f0ef94a3d163 2024-11-13T19:45:16,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527176324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527176325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527176327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c4c139fb53e4bbb94f593ef430cb3ba is 50, key is test_row_0/B:col10/1731527116167/Put/seqid=0 2024-11-13T19:45:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527176332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742388_1564 (size=12301) 2024-11-13T19:45:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c4c139fb53e4bbb94f593ef430cb3ba 2024-11-13T19:45:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527176339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f616276d2ec74cbabb76052e02b38f61 is 50, key is test_row_0/C:col10/1731527116167/Put/seqid=0 2024-11-13T19:45:16,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742389_1565 (size=12301) 2024-11-13T19:45:16,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f616276d2ec74cbabb76052e02b38f61 2024-11-13T19:45:16,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e83c4c701db848748aa7f0ef94a3d163 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163 2024-11-13T19:45:16,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163, entries=200, sequenceid=355, filesize=14.4 K 2024-11-13T19:45:16,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c4c139fb53e4bbb94f593ef430cb3ba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba 2024-11-13T19:45:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba, entries=150, sequenceid=355, filesize=12.0 K 2024-11-13T19:45:16,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f616276d2ec74cbabb76052e02b38f61 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61 2024-11-13T19:45:16,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61, entries=150, sequenceid=355, filesize=12.0 K 2024-11-13T19:45:16,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e12f682899aa55a4d182e84dc252f667 in 107ms, sequenceid=355, compaction requested=true 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:45:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:16,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:16,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b8f592bfbc8a4d55b83da724e3b81584 is 50, key is test_row_0/A:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742390_1566 (size=14741) 2024-11-13T19:45:16,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b8f592bfbc8a4d55b83da724e3b81584 2024-11-13T19:45:16,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c83ebf7802b54158b4a6a9004c693d66 is 50, key is test_row_0/B:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742391_1567 (size=12301) 2024-11-13T19:45:16,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c83ebf7802b54158b4a6a9004c693d66 2024-11-13T19:45:16,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/04e1b9b7393644e98edca711c6c777ed is 50, key is test_row_0/C:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527176506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527176508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527176510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742392_1568 (size=12301) 2024-11-13T19:45:16,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/04e1b9b7393644e98edca711c6c777ed 2024-11-13T19:45:16,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527176512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527176514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/b8f592bfbc8a4d55b83da724e3b81584 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584 2024-11-13T19:45:16,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584, entries=200, sequenceid=370, filesize=14.4 K 2024-11-13T19:45:16,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c83ebf7802b54158b4a6a9004c693d66 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66 2024-11-13T19:45:16,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66, entries=150, sequenceid=370, filesize=12.0 K 2024-11-13T19:45:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/04e1b9b7393644e98edca711c6c777ed as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed 2024-11-13T19:45:16,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed, entries=150, sequenceid=370, filesize=12.0 K 2024-11-13T19:45:16,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e12f682899aa55a4d182e84dc252f667 in 102ms, sequenceid=370, compaction requested=true 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:16,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-13T19:45:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:16,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T19:45:16,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:16,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ab65f2519e6b4227a62be2561a485d1b is 50, key is test_row_0/A:col10/1731527116642/Put/seqid=0 2024-11-13T19:45:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742393_1569 (size=14741) 2024-11-13T19:45:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527176676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,687 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4b9a9b0981ea44cb85d97338530cad38 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4b9a9b0981ea44cb85d97338530cad38 2024-11-13T19:45:16,687 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d1b97b55e1e04afe96a6d9a07848244d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d1b97b55e1e04afe96a6d9a07848244d 2024-11-13T19:45:16,693 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into d1b97b55e1e04afe96a6d9a07848244d(size=12.4 K), total size for store is 36.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:16,693 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,693 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527116245; duration=0sec 2024-11-13T19:45:16,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,694 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-13T19:45:16,694 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527176678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,694 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-13T19:45:16,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527176680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,695 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:16,695 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:16,695 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. because compaction request was cancelled 2024-11-13T19:45:16,695 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:16,695 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:45:16,695 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 4b9a9b0981ea44cb85d97338530cad38(size=12.4 K), total size for store is 41.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:16,696 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,696 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527116245; duration=0sec 2024-11-13T19:45:16,696 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-13T19:45:16,696 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:16,696 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61881 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-13T19:45:16,698 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. because compaction request was cancelled 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:16,698 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9a8d20d9c877433f94fb7668a2ac3ea6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=60.4 K 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:16,698 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:16,699 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8d20d9c877433f94fb7668a2ac3ea6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731527114047 2024-11-13T19:45:16,699 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42159 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:16,699 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:16,699 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,699 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4b9a9b0981ea44cb85d97338530cad38, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=41.2 K 2024-11-13T19:45:16,700 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9a9b0981ea44cb85d97338530cad38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731527115464 2024-11-13T19:45:16,702 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 73763b2318d44096833c3aeb52c8a677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731527114748 2024-11-13T19:45:16,702 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83c4c701db848748aa7f0ef94a3d163, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527116160 2024-11-13T19:45:16,702 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3d7be09627423db164dbccfc390c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731527115464 2024-11-13T19:45:16,702 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f592bfbc8a4d55b83da724e3b81584, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:16,703 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f616276d2ec74cbabb76052e02b38f61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527116160 2024-11-13T19:45:16,703 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e1b9b7393644e98edca711c6c777ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527176683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527176683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,724 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:16,724 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5064b7749b9d41f882932ca957b988b8 is 50, key is test_row_0/A:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,726 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:16,727 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14 is 50, key is test_row_0/C:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742394_1570 (size=12779) 2024-11-13T19:45:16,749 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/5064b7749b9d41f882932ca957b988b8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5064b7749b9d41f882932ca957b988b8 2024-11-13T19:45:16,757 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 5064b7749b9d41f882932ca957b988b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:16,757 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,757 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527116399; duration=0sec 2024-11-13T19:45:16,757 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:45:16,757 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:16,757 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:16,758 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:16,758 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:16,758 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,758 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d1b97b55e1e04afe96a6d9a07848244d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.4 K 2024-11-13T19:45:16,759 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b97b55e1e04afe96a6d9a07848244d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731527115464 2024-11-13T19:45:16,759 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4c139fb53e4bbb94f593ef430cb3ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731527116160 2024-11-13T19:45:16,760 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83ebf7802b54158b4a6a9004c693d66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:16,771 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:16,772 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/7343ba4e83e14e90b5d834c72bfc52d9 is 50, key is test_row_0/B:col10/1731527116323/Put/seqid=0 2024-11-13T19:45:16,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742395_1571 (size=12847) 2024-11-13T19:45:16,783 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14 2024-11-13T19:45:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:45:16,785 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:16,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T19:45:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:16,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:16,788 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:16,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:16,791 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 4fe5b49fe2cd4b3d9c49f54dc56f7e14(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:16,792 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=11, startTime=1731527116562; duration=0sec 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 3 compacting, 0 eligible, 16 blocking 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. because compaction request was cancelled 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T19:45:16,792 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:16,793 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:16,793 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. because compaction request was cancelled 2024-11-13T19:45:16,793 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:16,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742396_1572 (size=12779) 2024-11-13T19:45:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527176793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527176803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527176803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527176814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527176814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:16,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:16,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527177013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527177024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527177024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527177033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527177033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ab65f2519e6b4227a62be2561a485d1b 2024-11-13T19:45:17,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3e98eebd6ea6467397c0238ccd6448b1 is 50, key is test_row_0/B:col10/1731527116642/Put/seqid=0 2024-11-13T19:45:17,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742397_1573 (size=12301) 2024-11-13T19:45:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:17,208 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/7343ba4e83e14e90b5d834c72bfc52d9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7343ba4e83e14e90b5d834c72bfc52d9 2024-11-13T19:45:17,218 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 7343ba4e83e14e90b5d834c72bfc52d9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:17,218 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:17,218 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527116399; duration=0sec 2024-11-13T19:45:17,218 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:17,219 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:17,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527177334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527177343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527177352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527177352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527177366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:17,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3e98eebd6ea6467397c0238ccd6448b1 2024-11-13T19:45:17,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f84a4801194f480d82cab43902086198 is 50, key is test_row_0/C:col10/1731527116642/Put/seqid=0 2024-11-13T19:45:17,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742398_1574 (size=12301) 2024-11-13T19:45:17,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527177843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:17,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:17,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:17,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527177874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527177874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527177874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527177883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:17,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f84a4801194f480d82cab43902086198 2024-11-13T19:45:17,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ab65f2519e6b4227a62be2561a485d1b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b 2024-11-13T19:45:17,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b, entries=200, sequenceid=388, filesize=14.4 K 2024-11-13T19:45:17,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3e98eebd6ea6467397c0238ccd6448b1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1 2024-11-13T19:45:17,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1, entries=150, sequenceid=388, filesize=12.0 K 2024-11-13T19:45:17,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f84a4801194f480d82cab43902086198 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198 2024-11-13T19:45:17,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198, entries=150, sequenceid=388, filesize=12.0 K 2024-11-13T19:45:17,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e12f682899aa55a4d182e84dc252f667 in 1309ms, sequenceid=388, compaction requested=false 2024-11-13T19:45:17,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:18,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:45:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:18,030 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/30b58a8c9a14492483636f2e07004aa3 is 50, key is test_row_0/A:col10/1731527116680/Put/seqid=0 2024-11-13T19:45:18,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742399_1575 (size=9857) 2024-11-13T19:45:18,446 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/30b58a8c9a14492483636f2e07004aa3 2024-11-13T19:45:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3cd8c3bfb1774f48b9e0744dbe829fb8 is 50, key is test_row_0/B:col10/1731527116680/Put/seqid=0 2024-11-13T19:45:18,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742400_1576 (size=9857) 2024-11-13T19:45:18,501 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3cd8c3bfb1774f48b9e0744dbe829fb8 2024-11-13T19:45:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/dfcc2f7fc1e149e4a30e89c49a2d145d is 50, key is test_row_0/C:col10/1731527116680/Put/seqid=0 2024-11-13T19:45:18,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742401_1577 (size=9857) 2024-11-13T19:45:18,540 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/dfcc2f7fc1e149e4a30e89c49a2d145d 2024-11-13T19:45:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/30b58a8c9a14492483636f2e07004aa3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3 2024-11-13T19:45:18,564 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3, entries=100, sequenceid=407, filesize=9.6 K 2024-11-13T19:45:18,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3cd8c3bfb1774f48b9e0744dbe829fb8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8 2024-11-13T19:45:18,572 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8, entries=100, sequenceid=407, filesize=9.6 K 2024-11-13T19:45:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/dfcc2f7fc1e149e4a30e89c49a2d145d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d 2024-11-13T19:45:18,576 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d, entries=100, sequenceid=407, filesize=9.6 K 2024-11-13T19:45:18,577 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for e12f682899aa55a4d182e84dc252f667 in 548ms, sequenceid=407, compaction requested=true 2024-11-13T19:45:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T19:45:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T19:45:18,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T19:45:18,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-13T19:45:18,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8160 sec 2024-11-13T19:45:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:18,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:18,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:18,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/86e4c2f8676846389dd89561dae714ab is 50, key is test_row_0/A:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:18,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742402_1578 (size=12301) 2024-11-13T19:45:18,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/86e4c2f8676846389dd89561dae714ab 2024-11-13T19:45:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:45:18,944 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:18,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T19:45:18,952 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:18,953 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:18,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:18,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b4c787c1e605437098363e94fc22d78a is 50, key is test_row_0/B:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:18,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742403_1579 (size=12301) 2024-11-13T19:45:18,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b4c787c1e605437098363e94fc22d78a 2024-11-13T19:45:18,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527118938, max timestamp=1731527118963Num uniques 859; 2024-11-13T19:45:18,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:18,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527118938, max timestamp=1731527118963Num uniques 859; 2024-11-13T19:45:18,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:45:18,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527118938, max timestamp=1731527118963Num uniques 859; 2024-11-13T19:45:18,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:45:18,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/721025085143407280b0ed50a34447fc is 50, key is test_row_0/C:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742404_1580 (size=12301) 2024-11-13T19:45:19,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/721025085143407280b0ed50a34447fc 2024-11-13T19:45:19,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/86e4c2f8676846389dd89561dae714ab as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab 2024-11-13T19:45:19,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab, entries=150, sequenceid=417, filesize=12.0 K 2024-11-13T19:45:19,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b4c787c1e605437098363e94fc22d78a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a 2024-11-13T19:45:19,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a, entries=150, sequenceid=417, filesize=12.0 K 2024-11-13T19:45:19,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/721025085143407280b0ed50a34447fc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc 2024-11-13T19:45:19,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc, entries=150, sequenceid=417, filesize=12.0 K 2024-11-13T19:45:19,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=123.00 KB/125950 for e12f682899aa55a4d182e84dc252f667 in 124ms, sequenceid=417, compaction requested=true 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:19,024 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:19,024 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:19,025 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:19,025 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:19,025 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:19,025 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:19,025 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,025 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,025 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5064b7749b9d41f882932ca957b988b8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=48.5 K 2024-11-13T19:45:19,025 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7343ba4e83e14e90b5d834c72bfc52d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=46.1 K 2024-11-13T19:45:19,026 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7343ba4e83e14e90b5d834c72bfc52d9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:19,026 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5064b7749b9d41f882932ca957b988b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:19,026 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab65f2519e6b4227a62be2561a485d1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731527116507 2024-11-13T19:45:19,026 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e98eebd6ea6467397c0238ccd6448b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731527116507 2024-11-13T19:45:19,027 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b58a8c9a14492483636f2e07004aa3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731527116679 2024-11-13T19:45:19,027 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd8c3bfb1774f48b9e0744dbe829fb8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731527116679 2024-11-13T19:45:19,027 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c787c1e605437098363e94fc22d78a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:19,027 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e4c2f8676846389dd89561dae714ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:19,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:19,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/8b8699a71f21498ba40dc2a4b38d8306 is 50, key is test_row_0/A:col10/1731527119026/Put/seqid=0 2024-11-13T19:45:19,052 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:19,052 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d7d5f8b660d14965b649022875e30cc5 is 50, key is test_row_0/B:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:19,068 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,068 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4221c6cc3895455b87398e3c8195b736 is 50, key is test_row_0/A:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527179046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527179065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527179068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527179068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527179069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742405_1581 (size=14741) 2024-11-13T19:45:19,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/8b8699a71f21498ba40dc2a4b38d8306 2024-11-13T19:45:19,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742406_1582 (size=12915) 2024-11-13T19:45:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742407_1583 (size=12915) 2024-11-13T19:45:19,131 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/4221c6cc3895455b87398e3c8195b736 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4221c6cc3895455b87398e3c8195b736 2024-11-13T19:45:19,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ee8512f2a2e64c43a903974006519810 is 50, key is test_row_0/B:col10/1731527119026/Put/seqid=0 2024-11-13T19:45:19,141 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 4221c6cc3895455b87398e3c8195b736(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:19,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:19,141 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=12, startTime=1731527119024; duration=0sec 2024-11-13T19:45:19,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:19,141 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:19,142 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:19,143 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:19,143 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:19,143 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,143 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=46.2 K 2024-11-13T19:45:19,143 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe5b49fe2cd4b3d9c49f54dc56f7e14, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731527116323 2024-11-13T19:45:19,144 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84a4801194f480d82cab43902086198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731527116507 2024-11-13T19:45:19,145 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfcc2f7fc1e149e4a30e89c49a2d145d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731527116679 2024-11-13T19:45:19,145 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721025085143407280b0ed50a34447fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:19,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742408_1584 (size=12301) 2024-11-13T19:45:19,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ee8512f2a2e64c43a903974006519810 2024-11-13T19:45:19,159 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:19,160 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/e54e3c5349cd4925b55e1828a23e13e2 is 50, key is test_row_0/C:col10/1731527118894/Put/seqid=0 2024-11-13T19:45:19,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742409_1585 (size=12983) 2024-11-13T19:45:19,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/94836c538a794f2cb336686b94f6b2e9 is 50, key is test_row_0/C:col10/1731527119026/Put/seqid=0 2024-11-13T19:45:19,181 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/e54e3c5349cd4925b55e1828a23e13e2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/e54e3c5349cd4925b55e1828a23e13e2 2024-11-13T19:45:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527179173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,186 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into e54e3c5349cd4925b55e1828a23e13e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:19,186 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:19,186 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=12, startTime=1731527119024; duration=0sec 2024-11-13T19:45:19,186 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:19,186 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:19,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742410_1586 (size=12301) 2024-11-13T19:45:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527179194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527179194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527179194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527179203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:19,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527179393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527179403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527179414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527179414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527179424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d7d5f8b660d14965b649022875e30cc5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d7d5f8b660d14965b649022875e30cc5 2024-11-13T19:45:19,525 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into d7d5f8b660d14965b649022875e30cc5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:19,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:19,525 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=12, startTime=1731527119024; duration=0sec 2024-11-13T19:45:19,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:19,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:19,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:19,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/94836c538a794f2cb336686b94f6b2e9 2024-11-13T19:45:19,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/8b8699a71f21498ba40dc2a4b38d8306 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306 2024-11-13T19:45:19,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306, entries=200, sequenceid=452, filesize=14.4 K 2024-11-13T19:45:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/ee8512f2a2e64c43a903974006519810 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810 2024-11-13T19:45:19,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T19:45:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/94836c538a794f2cb336686b94f6b2e9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9 2024-11-13T19:45:19,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T19:45:19,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=60.38 KB/61830 for e12f682899aa55a4d182e84dc252f667 in 598ms, sequenceid=452, compaction requested=false 2024-11-13T19:45:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:19,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:19,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/83a71291fc8047cdb66b41341ec5ae35 is 50, key is test_row_0/A:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:19,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742411_1587 (size=12301) 2024-11-13T19:45:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527179773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527179774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527179774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527179788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527179789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:19,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:19,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527179894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527179894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527179894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527179903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:19,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527179913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527180104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527180104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527180113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527180123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527180134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/83a71291fc8047cdb66b41341ec5ae35 2024-11-13T19:45:20,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76dbc28e036a4427aead034e1405e2cd is 50, key is test_row_0/B:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:20,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742412_1588 (size=12301) 2024-11-13T19:45:20,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76dbc28e036a4427aead034e1405e2cd 2024-11-13T19:45:20,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7e88459b21f94b23b81c51bd331e89b4 is 50, key is test_row_0/C:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:20,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742413_1589 (size=12301) 2024-11-13T19:45:20,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7e88459b21f94b23b81c51bd331e89b4 2024-11-13T19:45:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:20,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/83a71291fc8047cdb66b41341ec5ae35 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35 2024-11-13T19:45:20,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T19:45:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/76dbc28e036a4427aead034e1405e2cd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd 2024-11-13T19:45:20,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T19:45:20,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7e88459b21f94b23b81c51bd331e89b4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4 2024-11-13T19:45:20,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T19:45:20,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e12f682899aa55a4d182e84dc252f667 in 491ms, sequenceid=468, compaction requested=true 2024-11-13T19:45:20,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:20,213 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:20,213 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:20,213 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:20,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:20,214 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,214 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4221c6cc3895455b87398e3c8195b736, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.0 K 2024-11-13T19:45:20,214 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:20,214 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:20,214 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,214 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d7d5f8b660d14965b649022875e30cc5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.6 K 2024-11-13T19:45:20,214 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4221c6cc3895455b87398e3c8195b736, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:20,215 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d5f8b660d14965b649022875e30cc5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:20,215 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8699a71f21498ba40dc2a4b38d8306, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731527119008 2024-11-13T19:45:20,215 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8512f2a2e64c43a903974006519810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731527119008 2024-11-13T19:45:20,215 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a71291fc8047cdb66b41341ec5ae35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:20,216 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 76dbc28e036a4427aead034e1405e2cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:20,232 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:20,235 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e4f448c9994342c1b7d8e7b37b5564a0 is 50, key is test_row_0/A:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:20,246 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:20,247 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/145792158c044b62946c4b41d861ee43 is 50, key is test_row_0/B:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742414_1590 (size=13017) 2024-11-13T19:45:20,281 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e4f448c9994342c1b7d8e7b37b5564a0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e4f448c9994342c1b7d8e7b37b5564a0 2024-11-13T19:45:20,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742415_1591 (size=13017) 2024-11-13T19:45:20,291 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/145792158c044b62946c4b41d861ee43 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/145792158c044b62946c4b41d861ee43 2024-11-13T19:45:20,292 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into e4f448c9994342c1b7d8e7b37b5564a0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:20,292 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:20,292 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527120213; duration=0sec 2024-11-13T19:45:20,292 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:20,292 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:20,292 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:20,295 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:20,295 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:20,295 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,295 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/e54e3c5349cd4925b55e1828a23e13e2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.7 K 2024-11-13T19:45:20,296 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54e3c5349cd4925b55e1828a23e13e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731527118870 2024-11-13T19:45:20,297 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94836c538a794f2cb336686b94f6b2e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731527119008 2024-11-13T19:45:20,297 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e88459b21f94b23b81c51bd331e89b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:20,299 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 145792158c044b62946c4b41d861ee43(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:20,299 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:20,299 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527120213; duration=0sec 2024-11-13T19:45:20,299 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:20,299 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:20,306 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:20,306 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/b9bfc05679764bc4a8d7309d5b45021a is 50, key is test_row_0/C:col10/1731527119067/Put/seqid=0 2024-11-13T19:45:20,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742416_1592 (size=13085) 2024-11-13T19:45:20,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:45:20,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,341 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f1dfc3bbdff04a21a69ef1663ff77776 is 50, key is test_row_0/A:col10/1731527119787/Put/seqid=0 2024-11-13T19:45:20,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742417_1593 (size=12301) 2024-11-13T19:45:20,359 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f1dfc3bbdff04a21a69ef1663ff77776 2024-11-13T19:45:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/71357777b99c461ab79c582679c552b7 is 50, key is test_row_0/B:col10/1731527119787/Put/seqid=0 2024-11-13T19:45:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742418_1594 (size=12301) 2024-11-13T19:45:20,377 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/71357777b99c461ab79c582679c552b7 2024-11-13T19:45:20,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/b27b935bfcdf450bb12d08506cb08bb8 is 50, key is test_row_0/C:col10/1731527119787/Put/seqid=0 2024-11-13T19:45:20,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742419_1595 (size=12301) 2024-11-13T19:45:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:20,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527180445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527180450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527180451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527180454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527180454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527180564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527180564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527180573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527180573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,725 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/b9bfc05679764bc4a8d7309d5b45021a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b9bfc05679764bc4a8d7309d5b45021a 2024-11-13T19:45:20,734 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into b9bfc05679764bc4a8d7309d5b45021a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:20,734 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:20,734 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527120213; duration=0sec 2024-11-13T19:45:20,734 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:20,734 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527180783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527180783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,796 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/b27b935bfcdf450bb12d08506cb08bb8 2024-11-13T19:45:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527180794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527180794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:20,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/f1dfc3bbdff04a21a69ef1663ff77776 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776 2024-11-13T19:45:20,806 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T19:45:20,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/71357777b99c461ab79c582679c552b7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7 2024-11-13T19:45:20,810 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T19:45:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/b27b935bfcdf450bb12d08506cb08bb8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8 2024-11-13T19:45:20,815 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T19:45:20,815 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e12f682899aa55a4d182e84dc252f667 in 474ms, sequenceid=489, compaction requested=false 2024-11-13T19:45:20,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T19:45:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T19:45:20,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T19:45:20,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-13T19:45:20,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8720 sec 2024-11-13T19:45:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:20,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a81dbf7c1ec04fc38a2d7eb7f43949bc is 50, key is test_row_0/A:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:20,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742420_1596 (size=14741) 2024-11-13T19:45:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:45:21,104 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:21,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T19:45:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:45:21,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:21,108 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:21,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:21,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527181106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527181131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527181132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527181134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527181134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:45:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527181244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T19:45:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527181254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a81dbf7c1ec04fc38a2d7eb7f43949bc 2024-11-13T19:45:21,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d9a3ff04d7254d9db412237c2b53b56b is 50, key is test_row_0/B:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:21,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T19:45:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:45:21,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742421_1597 (size=12301) 2024-11-13T19:45:21,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d9a3ff04d7254d9db412237c2b53b56b 2024-11-13T19:45:21,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a65ee6b162c44f48af89180e40a68b81 is 50, key is test_row_0/C:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:21,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742422_1598 (size=12301) 2024-11-13T19:45:21,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a65ee6b162c44f48af89180e40a68b81 2024-11-13T19:45:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527181465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527181465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527181474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527181475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a81dbf7c1ec04fc38a2d7eb7f43949bc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc 2024-11-13T19:45:21,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527181485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc, entries=200, sequenceid=504, filesize=14.4 K 2024-11-13T19:45:21,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/d9a3ff04d7254d9db412237c2b53b56b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b 2024-11-13T19:45:21,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T19:45:21,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a65ee6b162c44f48af89180e40a68b81 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81 2024-11-13T19:45:21,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T19:45:21,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e12f682899aa55a4d182e84dc252f667 in 542ms, sequenceid=504, compaction requested=true 2024-11-13T19:45:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:21,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:21,524 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:21,524 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:21,524 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,524 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e4f448c9994342c1b7d8e7b37b5564a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.1 K 2024-11-13T19:45:21,524 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f448c9994342c1b7d8e7b37b5564a0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:21,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:21,525 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1dfc3bbdff04a21a69ef1663ff77776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731527119771 2024-11-13T19:45:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:21,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:21,526 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81dbf7c1ec04fc38a2d7eb7f43949bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120435 2024-11-13T19:45:21,535 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:21,535 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:21,535 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,535 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/145792158c044b62946c4b41d861ee43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.7 K 2024-11-13T19:45:21,537 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 145792158c044b62946c4b41d861ee43, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:21,538 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 71357777b99c461ab79c582679c552b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731527119771 2024-11-13T19:45:21,538 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a3ff04d7254d9db412237c2b53b56b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120449 2024-11-13T19:45:21,556 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:21,557 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/dc9daf6a43e04a8898f2511b24aac20a is 50, key is test_row_0/A:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:21,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T19:45:21,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,578 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:21,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,579 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:21,580 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1ea11a64114949269136e02373dc3c34 is 50, key is test_row_0/B:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:21,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/190053d5537d4257a310e6df39eb5b07 is 50, key is test_row_0/A:col10/1731527121104/Put/seqid=0 2024-11-13T19:45:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742423_1599 (size=13119) 2024-11-13T19:45:21,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742424_1600 (size=13119) 2024-11-13T19:45:21,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742425_1601 (size=12301) 2024-11-13T19:45:21,599 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/190053d5537d4257a310e6df39eb5b07 2024-11-13T19:45:21,600 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/dc9daf6a43e04a8898f2511b24aac20a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc9daf6a43e04a8898f2511b24aac20a 2024-11-13T19:45:21,604 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/1ea11a64114949269136e02373dc3c34 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1ea11a64114949269136e02373dc3c34 2024-11-13T19:45:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/01043690024e494293dddcc1f9bb11ab is 50, key is test_row_0/B:col10/1731527121104/Put/seqid=0 2024-11-13T19:45:21,610 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into dc9daf6a43e04a8898f2511b24aac20a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:21,610 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:21,610 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527121523; duration=0sec 2024-11-13T19:45:21,610 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:21,610 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:21,610 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:21,612 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:21,612 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:21,613 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,613 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b9bfc05679764bc4a8d7309d5b45021a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.8 K 2024-11-13T19:45:21,613 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bfc05679764bc4a8d7309d5b45021a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731527119066 2024-11-13T19:45:21,613 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27b935bfcdf450bb12d08506cb08bb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731527119771 2024-11-13T19:45:21,614 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 1ea11a64114949269136e02373dc3c34(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:21,614 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:21,614 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527121525; duration=0sec 2024-11-13T19:45:21,614 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:21,614 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65ee6b162c44f48af89180e40a68b81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120449 2024-11-13T19:45:21,614 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742426_1602 (size=12301) 2024-11-13T19:45:21,619 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/01043690024e494293dddcc1f9bb11ab 2024-11-13T19:45:21,625 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:21,626 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/2a8dd99bb4ab488ca500349037438d6d is 50, key is test_row_0/C:col10/1731527120452/Put/seqid=0 2024-11-13T19:45:21,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/59ea7c05da48481ca5eea42eb562c95f is 50, key is test_row_0/C:col10/1731527121104/Put/seqid=0 2024-11-13T19:45:21,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742427_1603 (size=13187) 2024-11-13T19:45:21,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742428_1604 (size=12301) 2024-11-13T19:45:21,652 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/59ea7c05da48481ca5eea42eb562c95f 2024-11-13T19:45:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/190053d5537d4257a310e6df39eb5b07 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07 2024-11-13T19:45:21,665 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/2a8dd99bb4ab488ca500349037438d6d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/2a8dd99bb4ab488ca500349037438d6d 2024-11-13T19:45:21,669 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07, entries=150, sequenceid=523, filesize=12.0 K 2024-11-13T19:45:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/01043690024e494293dddcc1f9bb11ab as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab 2024-11-13T19:45:21,672 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 2a8dd99bb4ab488ca500349037438d6d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:21,672 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:21,672 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527121525; duration=0sec 2024-11-13T19:45:21,673 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:21,673 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:21,677 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab, entries=150, sequenceid=523, filesize=12.0 K 2024-11-13T19:45:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/59ea7c05da48481ca5eea42eb562c95f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f 2024-11-13T19:45:21,684 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f, entries=150, sequenceid=523, filesize=12.0 K 2024-11-13T19:45:21,685 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for e12f682899aa55a4d182e84dc252f667 in 107ms, sequenceid=523, compaction requested=false 2024-11-13T19:45:21,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:21,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T19:45:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T19:45:21,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T19:45:21,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 579 msec 2024-11-13T19:45:21,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 587 msec 2024-11-13T19:45:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:45:21,734 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:21,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T19:45:21,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:21,739 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:21,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:21,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:45:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:21,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/dc4ae120a7f74a91af94894247ba26e0 is 50, key is test_row_0/A:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742429_1605 (size=14741) 2024-11-13T19:45:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527181856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527121848, max timestamp=1731527121868Num uniques 859; 2024-11-13T19:45:21,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:45:21,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731527121848, max timestamp=1731527121868Num uniques 859; 2024-11-13T19:45:21,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:45:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731527121848, max timestamp=1731527121868Num uniques 859; 2024-11-13T19:45:21,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T19:45:21,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:45:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:21,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527181915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527181928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527181932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527181933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:21,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527181984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:45:22,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527182043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527182044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527182054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527182054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:45:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527182203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/dc4ae120a7f74a91af94894247ba26e0 2024-11-13T19:45:22,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f607f698d50b48da8f0c129b17776e91 is 50, key is test_row_0/B:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742430_1606 (size=12301) 2024-11-13T19:45:22,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f607f698d50b48da8f0c129b17776e91 2024-11-13T19:45:22,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527182254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527182254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/004d2871c34f452cbc8693875a514deb is 50, key is test_row_0/C:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742431_1607 (size=12301) 2024-11-13T19:45:22,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/004d2871c34f452cbc8693875a514deb 2024-11-13T19:45:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527182279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527182279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/dc4ae120a7f74a91af94894247ba26e0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0 2024-11-13T19:45:22,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0, entries=200, sequenceid=537, filesize=14.4 K 2024-11-13T19:45:22,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f607f698d50b48da8f0c129b17776e91 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91 2024-11-13T19:45:22,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91, entries=150, sequenceid=537, filesize=12.0 K 2024-11-13T19:45:22,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/004d2871c34f452cbc8693875a514deb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb 2024-11-13T19:45:22,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb, entries=150, sequenceid=537, filesize=12.0 K 2024-11-13T19:45:22,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e12f682899aa55a4d182e84dc252f667 in 502ms, sequenceid=537, compaction requested=true 2024-11-13T19:45:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:22,311 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:22,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:22,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:22,312 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:22,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:22,312 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:22,312 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:22,312 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,313 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc9daf6a43e04a8898f2511b24aac20a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.2 K 2024-11-13T19:45:22,313 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9daf6a43e04a8898f2511b24aac20a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120449 2024-11-13T19:45:22,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:22,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:22,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:22,313 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:22,313 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:22,313 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,314 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1ea11a64114949269136e02373dc3c34, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.8 K 2024-11-13T19:45:22,314 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 190053d5537d4257a310e6df39eb5b07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731527121104 2024-11-13T19:45:22,314 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea11a64114949269136e02373dc3c34, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120449 2024-11-13T19:45:22,314 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc4ae120a7f74a91af94894247ba26e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:22,314 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 01043690024e494293dddcc1f9bb11ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731527121104 2024-11-13T19:45:22,315 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f607f698d50b48da8f0c129b17776e91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:22,325 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:22,326 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ee65e590bdf044c384eb77f049516522 is 50, key is test_row_0/A:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:22,331 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#526 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:22,331 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/9394d2ae95084ea183814f22752172e3 is 50, key is test_row_0/B:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:22,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742432_1608 (size=13221) 2024-11-13T19:45:22,340 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ee65e590bdf044c384eb77f049516522 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ee65e590bdf044c384eb77f049516522 2024-11-13T19:45:22,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742433_1609 (size=13221) 2024-11-13T19:45:22,345 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into ee65e590bdf044c384eb77f049516522(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:22,345 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:22,345 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527122311; duration=0sec 2024-11-13T19:45:22,345 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:22,346 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:22,346 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:22,347 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:22,347 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:22,348 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,348 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/9394d2ae95084ea183814f22752172e3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/9394d2ae95084ea183814f22752172e3 2024-11-13T19:45:22,348 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/2a8dd99bb4ab488ca500349037438d6d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.9 K 2024-11-13T19:45:22,348 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a8dd99bb4ab488ca500349037438d6d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731527120449 2024-11-13T19:45:22,348 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ea7c05da48481ca5eea42eb562c95f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731527121104 2024-11-13T19:45:22,349 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004d2871c34f452cbc8693875a514deb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:22,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:45:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:22,351 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-13T19:45:22,353 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into 9394d2ae95084ea183814f22752172e3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:22,353 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:22,353 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527122312; duration=0sec 2024-11-13T19:45:22,353 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:22,353 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:22,358 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 is 50, key is test_row_0/A:col10/1731527121929/Put/seqid=0 2024-11-13T19:45:22,359 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7612439f492047de88a8691475455250 is 50, key is test_row_0/C:col10/1731527121795/Put/seqid=0 2024-11-13T19:45:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742434_1610 (size=13289) 2024-11-13T19:45:22,382 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/7612439f492047de88a8691475455250 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7612439f492047de88a8691475455250 2024-11-13T19:45:22,388 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 7612439f492047de88a8691475455250(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:22,388 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:22,388 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527122313; duration=0sec 2024-11-13T19:45:22,388 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:22,388 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:22,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742435_1611 (size=12301) 2024-11-13T19:45:22,390 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 2024-11-13T19:45:22,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f80e3dfba4d04497800092f8dc33007c is 50, key is test_row_0/B:col10/1731527121929/Put/seqid=0 2024-11-13T19:45:22,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742436_1612 (size=12301) 2024-11-13T19:45:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:22,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:22,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527182586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527182587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527182594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527182594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527182600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527182713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527182713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527182724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,811 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f80e3dfba4d04497800092f8dc33007c 2024-11-13T19:45:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0defb7658d834c33b08d589b4ea1d90e is 50, key is test_row_0/C:col10/1731527121929/Put/seqid=0 2024-11-13T19:45:22,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742437_1613 (size=12301) 2024-11-13T19:45:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527182924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527182925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:22,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527182943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527183114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527183114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527183243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527183244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,265 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0defb7658d834c33b08d589b4ea1d90e 2024-11-13T19:45:23,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 2024-11-13T19:45:23,271 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3, entries=150, sequenceid=573, filesize=12.0 K 2024-11-13T19:45:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f80e3dfba4d04497800092f8dc33007c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c 2024-11-13T19:45:23,275 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c, entries=150, sequenceid=573, filesize=12.0 K 2024-11-13T19:45:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/0defb7658d834c33b08d589b4ea1d90e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e 2024-11-13T19:45:23,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527183264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,280 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e, entries=150, sequenceid=573, filesize=12.0 K 2024-11-13T19:45:23,280 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for e12f682899aa55a4d182e84dc252f667 in 929ms, sequenceid=573, compaction requested=false 2024-11-13T19:45:23,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:23,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:23,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T19:45:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T19:45:23,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T19:45:23,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-13T19:45:23,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5510 sec 2024-11-13T19:45:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:23,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:23,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:23,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/314a4727fe9c47ef948531f83f23f828 is 50, key is test_row_0/A:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742438_1614 (size=14741) 2024-11-13T19:45:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527183849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527183857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527183860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:45:23,884 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:23,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T19:45:23,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:23,886 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527183963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527183984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527183984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:45:24,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527184123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,133 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_1' in e12f682899aa55a4d182e84dc252f667 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527184133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,142 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in e12f682899aa55a4d182e84dc252f667 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527184174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/314a4727fe9c47ef948531f83f23f828 2024-11-13T19:45:24,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:45:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2bf6e21bb79247a586d8203a57482963 is 50, key is test_row_0/B:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742439_1615 (size=12301) 2024-11-13T19:45:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:24,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2bf6e21bb79247a586d8203a57482963 2024-11-13T19:45:24,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527184204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527184204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a9f7ea5e27934ce1a808b94ecc2fb627 is 50, key is test_row_0/C:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:24,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742440_1616 (size=12301) 2024-11-13T19:45:24,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:45:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527184493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:24,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:45:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:24,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527184524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527184524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:24,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a9f7ea5e27934ce1a808b94ecc2fb627 2024-11-13T19:45:24,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/314a4727fe9c47ef948531f83f23f828 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828 2024-11-13T19:45:24,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828, entries=200, sequenceid=587, filesize=14.4 K 2024-11-13T19:45:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/2bf6e21bb79247a586d8203a57482963 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963 2024-11-13T19:45:24,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963, entries=150, sequenceid=587, filesize=12.0 K 2024-11-13T19:45:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a9f7ea5e27934ce1a808b94ecc2fb627 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627 2024-11-13T19:45:24,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627, entries=150, sequenceid=587, filesize=12.0 K 2024-11-13T19:45:24,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e12f682899aa55a4d182e84dc252f667 in 884ms, sequenceid=587, compaction requested=true 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:24,655 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:24,655 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:24,656 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:24,656 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ee65e590bdf044c384eb77f049516522, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.3 K 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:24,656 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,656 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/9394d2ae95084ea183814f22752172e3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=36.9 K 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9394d2ae95084ea183814f22752172e3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:24,656 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee65e590bdf044c384eb77f049516522, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:24,657 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5b4e3b7d8942febb8cc0f8cb0ff2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731527121890 2024-11-13T19:45:24,657 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f80e3dfba4d04497800092f8dc33007c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731527121890 2024-11-13T19:45:24,661 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf6e21bb79247a586d8203a57482963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:24,662 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 314a4727fe9c47ef948531f83f23f828, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:24,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:45:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,670 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:24,679 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:24,680 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/3808d6e69009487f8f3ab0a0d71efaf7 is 50, key is test_row_0/A:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:24,691 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:24,692 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c0302234ea80468592ada845dfae65b2 is 50, key is test_row_0/B:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/23208ca51d4f414593657ca01d2a4a7e is 50, key is test_row_0/A:col10/1731527123834/Put/seqid=0 2024-11-13T19:45:24,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742441_1617 (size=13323) 2024-11-13T19:45:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742442_1618 (size=13323) 2024-11-13T19:45:24,722 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/c0302234ea80468592ada845dfae65b2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c0302234ea80468592ada845dfae65b2 2024-11-13T19:45:24,732 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into c0302234ea80468592ada845dfae65b2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:24,732 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:24,732 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527124655; duration=0sec 2024-11-13T19:45:24,732 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:24,732 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:24,732 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:24,733 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:24,733 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:24,733 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:24,733 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7612439f492047de88a8691475455250, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=37.0 K 2024-11-13T19:45:24,733 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 7612439f492047de88a8691475455250, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731527121795 2024-11-13T19:45:24,734 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0defb7658d834c33b08d589b4ea1d90e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1731527121890 2024-11-13T19:45:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742443_1619 (size=12301) 2024-11-13T19:45:24,735 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f7ea5e27934ce1a808b94ecc2fb627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:24,755 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#537 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:24,756 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a684354fcc8b4ade81a5d12337e3d0f6 is 50, key is test_row_0/C:col10/1731527123769/Put/seqid=0 2024-11-13T19:45:24,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742444_1620 (size=13391) 2024-11-13T19:45:24,779 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/a684354fcc8b4ade81a5d12337e3d0f6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a684354fcc8b4ade81a5d12337e3d0f6 2024-11-13T19:45:24,789 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into a684354fcc8b4ade81a5d12337e3d0f6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:24,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:24,789 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527124655; duration=0sec 2024-11-13T19:45:24,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:24,789 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527185061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527185067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527185067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,114 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/3808d6e69009487f8f3ab0a0d71efaf7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3808d6e69009487f8f3ab0a0d71efaf7 2024-11-13T19:45:25,120 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into 3808d6e69009487f8f3ab0a0d71efaf7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:25,120 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:25,120 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527124655; duration=0sec 2024-11-13T19:45:25,120 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:25,120 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:25,135 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/23208ca51d4f414593657ca01d2a4a7e 2024-11-13T19:45:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c716eed186e4313b04a42c362f2860f is 50, key is test_row_0/B:col10/1731527123834/Put/seqid=0 2024-11-13T19:45:25,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742445_1621 (size=12301) 2024-11-13T19:45:25,162 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c716eed186e4313b04a42c362f2860f 2024-11-13T19:45:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/115cb319d21e4256bf685a1592ec8148 is 50, key is test_row_0/C:col10/1731527123834/Put/seqid=0 2024-11-13T19:45:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527185174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742446_1622 (size=12301) 2024-11-13T19:45:25,187 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/115cb319d21e4256bf685a1592ec8148 2024-11-13T19:45:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527185184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/23208ca51d4f414593657ca01d2a4a7e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e 2024-11-13T19:45:25,199 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e, entries=150, sequenceid=607, filesize=12.0 K 2024-11-13T19:45:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c716eed186e4313b04a42c362f2860f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f 2024-11-13T19:45:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527185193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,205 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f, entries=150, sequenceid=607, filesize=12.0 K 2024-11-13T19:45:25,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/115cb319d21e4256bf685a1592ec8148 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148 2024-11-13T19:45:25,210 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148, entries=150, sequenceid=607, filesize=12.0 K 2024-11-13T19:45:25,210 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e12f682899aa55a4d182e84dc252f667 in 540ms, sequenceid=607, compaction requested=false 2024-11-13T19:45:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T19:45:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T19:45:25,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T19:45:25,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-13T19:45:25,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3370 sec 2024-11-13T19:45:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:25,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:25,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ddf828fc7351440082c1e1110fc689d9 is 50, key is test_row_0/A:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742447_1623 (size=14741) 2024-11-13T19:45:25,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ddf828fc7351440082c1e1110fc689d9 2024-11-13T19:45:25,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3463d63caa6b408db966636a1bcb00c3 is 50, key is test_row_0/B:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742448_1624 (size=12301) 2024-11-13T19:45:25,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3463d63caa6b408db966636a1bcb00c3 2024-11-13T19:45:25,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9743b13b09ca428a80765079cc4bc3e5 is 50, key is test_row_0/C:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527185462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742449_1625 (size=12301) 2024-11-13T19:45:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527185472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527185478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527185584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527185595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527185604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527185803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527185813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527185823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:25,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9743b13b09ca428a80765079cc4bc3e5 2024-11-13T19:45:25,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ddf828fc7351440082c1e1110fc689d9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9 2024-11-13T19:45:25,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9, entries=200, sequenceid=623, filesize=14.4 K 2024-11-13T19:45:25,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3463d63caa6b408db966636a1bcb00c3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3 2024-11-13T19:45:25,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3, entries=150, sequenceid=623, filesize=12.0 K 2024-11-13T19:45:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/9743b13b09ca428a80765079cc4bc3e5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5 2024-11-13T19:45:25,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5, entries=150, sequenceid=623, filesize=12.0 K 2024-11-13T19:45:25,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e12f682899aa55a4d182e84dc252f667 in 497ms, sequenceid=623, compaction requested=true 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:25,895 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:25,895 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:25,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:25,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:25,896 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:25,896 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:25,896 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:25,896 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:25,896 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:25,897 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:25,897 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3808d6e69009487f8f3ab0a0d71efaf7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=39.4 K 2024-11-13T19:45:25,897 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c0302234ea80468592ada845dfae65b2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=37.0 K 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3808d6e69009487f8f3ab0a0d71efaf7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c0302234ea80468592ada845dfae65b2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23208ca51d4f414593657ca01d2a4a7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731527123834 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c716eed186e4313b04a42c362f2860f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731527123834 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf828fc7351440082c1e1110fc689d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:25,897 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3463d63caa6b408db966636a1bcb00c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:25,903 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:25,903 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/fc1821e151244a5ba9cb6cab876768d0 is 50, key is test_row_0/B:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,904 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#544 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:25,904 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ecc3d0012d3a46a289dcb5f4aa7686bd is 50, key is test_row_0/A:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742450_1626 (size=13425) 2024-11-13T19:45:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742451_1627 (size=13425) 2024-11-13T19:45:25,919 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ecc3d0012d3a46a289dcb5f4aa7686bd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ecc3d0012d3a46a289dcb5f4aa7686bd 2024-11-13T19:45:25,923 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into ecc3d0012d3a46a289dcb5f4aa7686bd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:25,923 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:25,923 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=13, startTime=1731527125895; duration=0sec 2024-11-13T19:45:25,923 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:25,923 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:25,923 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:25,924 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:25,924 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:25,924 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:25,924 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a684354fcc8b4ade81a5d12337e3d0f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=37.1 K 2024-11-13T19:45:25,925 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a684354fcc8b4ade81a5d12337e3d0f6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731527122583 2024-11-13T19:45:25,925 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 115cb319d21e4256bf685a1592ec8148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731527123834 2024-11-13T19:45:25,925 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9743b13b09ca428a80765079cc4bc3e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:25,935 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:25,935 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4adad769f608469993a511db61284221 is 50, key is test_row_0/C:col10/1731527125058/Put/seqid=0 2024-11-13T19:45:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742452_1628 (size=13493) 2024-11-13T19:45:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:45:26,033 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:26,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T19:45:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:26,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:26,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:26,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:26,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/aad375bfdd9040d89071b7f05fc74f3a is 50, key is test_row_0/A:col10/1731527125470/Put/seqid=0 2024-11-13T19:45:26,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742453_1629 (size=14741) 2024-11-13T19:45:26,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/aad375bfdd9040d89071b7f05fc74f3a 2024-11-13T19:45:26,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/18f6dd7c1e03483fa18ee8740253cc5b is 50, key is test_row_0/B:col10/1731527125470/Put/seqid=0 2024-11-13T19:45:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:26,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742454_1630 (size=12301) 2024-11-13T19:45:26,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/18f6dd7c1e03483fa18ee8740253cc5b 2024-11-13T19:45:26,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f0a69d26981a40a38ccf60d0d36b6d1c is 50, key is test_row_0/C:col10/1731527125470/Put/seqid=0 2024-11-13T19:45:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527186155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59554 deadline: 1731527186156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,168 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in e12f682899aa55a4d182e84dc252f667 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527186157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527186158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527186167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742455_1631 (size=12301) 2024-11-13T19:45:26,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:45:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527186274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527186274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527186274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527186283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,316 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/fc1821e151244a5ba9cb6cab876768d0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fc1821e151244a5ba9cb6cab876768d0 2024-11-13T19:45:26,320 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into fc1821e151244a5ba9cb6cab876768d0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:26,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:26,320 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=13, startTime=1731527125895; duration=0sec 2024-11-13T19:45:26,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:26,320 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:26,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:45:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,362 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4adad769f608469993a511db61284221 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4adad769f608469993a511db61284221 2024-11-13T19:45:26,367 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 4adad769f608469993a511db61284221(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:26,367 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:26,367 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=13, startTime=1731527125895; duration=0sec 2024-11-13T19:45:26,367 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:26,367 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:26,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527186493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527186493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527186493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:45:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527186504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f0a69d26981a40a38ccf60d0d36b6d1c 2024-11-13T19:45:26,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/aad375bfdd9040d89071b7f05fc74f3a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a 2024-11-13T19:45:26,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a, entries=200, sequenceid=644, filesize=14.4 K 2024-11-13T19:45:26,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/18f6dd7c1e03483fa18ee8740253cc5b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b 2024-11-13T19:45:26,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b, entries=150, sequenceid=644, filesize=12.0 K 2024-11-13T19:45:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/f0a69d26981a40a38ccf60d0d36b6d1c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c 2024-11-13T19:45:26,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c, entries=150, sequenceid=644, filesize=12.0 K 2024-11-13T19:45:26,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e12f682899aa55a4d182e84dc252f667 in 517ms, sequenceid=644, compaction requested=false 2024-11-13T19:45:26,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:26,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:26,666 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:26,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e3ba1ba6cf8141fd804dbdecd616c9cd is 50, key is test_row_0/A:col10/1731527126155/Put/seqid=0 2024-11-13T19:45:26,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742456_1632 (size=12301) 2024-11-13T19:45:26,690 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e3ba1ba6cf8141fd804dbdecd616c9cd 2024-11-13T19:45:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c3468e254324f20b68545acafa79b3f is 50, key is test_row_0/B:col10/1731527126155/Put/seqid=0 2024-11-13T19:45:26,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742457_1633 (size=12301) 2024-11-13T19:45:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:26,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. as already flushing 2024-11-13T19:45:26,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527186866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527186869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527186870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527186883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,942 INFO [Thread-2197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2197. 2024-11-13T19:45:26,942 DEBUG [Thread-2197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:26,942 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,943 INFO [Thread-2193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2193. 2024-11-13T19:45:26,943 DEBUG [Thread-2193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:26,943 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,944 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,944 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:26,944 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:26,951 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-11-13T19:45:26,951 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:26,951 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,951 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:26,956 INFO [Thread-2195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2195. 2024-11-13T19:45:26,956 DEBUG [Thread-2195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:26,956 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,956 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:26,959 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-11-13T19:45:26,959 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:26,959 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,959 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:26,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527186993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527186993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527186993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527187004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,146 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c3468e254324f20b68545acafa79b3f 2024-11-13T19:45:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 is 50, key is test_row_0/C:col10/1731527126155/Put/seqid=0 2024-11-13T19:45:27,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742458_1634 (size=12301) 2024-11-13T19:45:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527187204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527187204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527187204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527187213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59622 deadline: 1731527187514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59582 deadline: 1731527187514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59566 deadline: 1731527187514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59608 deadline: 1731527187523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 , the old value is region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., hostname=54d0784d1943,36227,1731526989835, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:45:27,555 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 2024-11-13T19:45:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/e3ba1ba6cf8141fd804dbdecd616c9cd as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd 2024-11-13T19:45:27,563 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd, entries=150, sequenceid=658, filesize=12.0 K 2024-11-13T19:45:27,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/5c3468e254324f20b68545acafa79b3f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f 2024-11-13T19:45:27,568 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f, entries=150, sequenceid=658, filesize=12.0 K 2024-11-13T19:45:27,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 2024-11-13T19:45:27,572 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57, entries=150, sequenceid=658, filesize=12.0 K 2024-11-13T19:45:27,572 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e12f682899aa55a4d182e84dc252f667 in 906ms, sequenceid=658, compaction requested=true 2024-11-13T19:45:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T19:45:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T19:45:27,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T19:45:27,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5370 sec 2024-11-13T19:45:27,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5420 sec 2024-11-13T19:45:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:28,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-13T19:45:28,027 INFO [Thread-2188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2188. 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:28,027 DEBUG [Thread-2188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:28,027 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:28,027 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:28,028 INFO [Thread-2186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2186. 2024-11-13T19:45:28,028 DEBUG [Thread-2186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:28,028 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,028 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:28,029 INFO [Thread-2184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2184. 2024-11-13T19:45:28,029 DEBUG [Thread-2184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:28,029 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:28,030 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/939ac56444d84e6289a565b035ad432f is 50, key is test_row_0/A:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:28,035 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-11-13T19:45:28,035 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:28,035 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,036 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:28,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742459_1635 (size=12301) 2024-11-13T19:45:28,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/939ac56444d84e6289a565b035ad432f 2024-11-13T19:45:28,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f71b36728c944162bf050e91a3bf096c is 50, key is test_row_0/B:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:28,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742460_1636 (size=12301) 2024-11-13T19:45:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:45:28,184 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:28,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f71b36728c944162bf050e91a3bf096c 2024-11-13T19:45:28,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/53cd8a7d4b784b4c8ea10ba4a61f6154 is 50, key is test_row_0/C:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:28,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742461_1637 (size=12301) 2024-11-13T19:45:28,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/53cd8a7d4b784b4c8ea10ba4a61f6154 2024-11-13T19:45:28,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/939ac56444d84e6289a565b035ad432f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f 2024-11-13T19:45:28,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f, entries=150, sequenceid=681, filesize=12.0 K 2024-11-13T19:45:28,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/f71b36728c944162bf050e91a3bf096c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c 2024-11-13T19:45:28,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c, entries=150, sequenceid=681, filesize=12.0 K 2024-11-13T19:45:28,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/53cd8a7d4b784b4c8ea10ba4a61f6154 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154 2024-11-13T19:45:28,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154, entries=150, sequenceid=681, filesize=12.0 K 2024-11-13T19:45:28,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=6.71 KB/6870 for e12f682899aa55a4d182e84dc252f667 in 850ms, sequenceid=681, compaction requested=true 2024-11-13T19:45:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:28,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:28,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e12f682899aa55a4d182e84dc252f667:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:28,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:28,878 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:28,878 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:28,878 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:28,878 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/B is initiating minor compaction (all files) 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/A is initiating minor compaction (all files) 2024-11-13T19:45:28,879 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/B in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:28,879 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/A in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:28,879 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fc1821e151244a5ba9cb6cab876768d0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=49.1 K 2024-11-13T19:45:28,879 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ecc3d0012d3a46a289dcb5f4aa7686bd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=51.5 K 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1821e151244a5ba9cb6cab876768d0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc3d0012d3a46a289dcb5f4aa7686bd, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f6dd7c1e03483fa18ee8740253cc5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731527125455 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad375bfdd9040d89071b7f05fc74f3a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731527125455 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3468e254324f20b68545acafa79b3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731527126153 2024-11-13T19:45:28,879 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ba1ba6cf8141fd804dbdecd616c9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731527126153 2024-11-13T19:45:28,880 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f71b36728c944162bf050e91a3bf096c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=681, earliestPutTs=1731527126865 2024-11-13T19:45:28,880 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939ac56444d84e6289a565b035ad432f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=681, earliestPutTs=1731527126865 2024-11-13T19:45:28,887 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#A#compaction#555 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:28,887 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ec025ffd267e496e82c21d3866dcd610 is 50, key is test_row_0/A:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:28,888 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#B#compaction#556 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:28,889 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b18bc2d241c045358ea0759ea37891a9 is 50, key is test_row_0/B:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:28,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742462_1638 (size=13561) 2024-11-13T19:45:28,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742463_1639 (size=13561) 2024-11-13T19:45:29,298 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/b18bc2d241c045358ea0759ea37891a9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b18bc2d241c045358ea0759ea37891a9 2024-11-13T19:45:29,298 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/ec025ffd267e496e82c21d3866dcd610 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ec025ffd267e496e82c21d3866dcd610 2024-11-13T19:45:29,302 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/A of e12f682899aa55a4d182e84dc252f667 into ec025ffd267e496e82c21d3866dcd610(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:29,302 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/B of e12f682899aa55a4d182e84dc252f667 into b18bc2d241c045358ea0759ea37891a9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:29,302 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/A, priority=12, startTime=1731527128877; duration=0sec 2024-11-13T19:45:29,302 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/B, priority=12, startTime=1731527128877; duration=0sec 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:B 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:A 2024-11-13T19:45:29,302 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:29,303 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:29,303 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): e12f682899aa55a4d182e84dc252f667/C is initiating minor compaction (all files) 2024-11-13T19:45:29,303 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e12f682899aa55a4d182e84dc252f667/C in TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:29,304 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4adad769f608469993a511db61284221, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp, totalSize=49.2 K 2024-11-13T19:45:29,304 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adad769f608469993a511db61284221, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1731527125058 2024-11-13T19:45:29,304 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a69d26981a40a38ccf60d0d36b6d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731527125455 2024-11-13T19:45:29,304 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3cbf2a8e6c4233a1f8f92dac84cf57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731527126153 2024-11-13T19:45:29,305 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cd8a7d4b784b4c8ea10ba4a61f6154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=681, earliestPutTs=1731527126865 2024-11-13T19:45:29,313 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e12f682899aa55a4d182e84dc252f667#C#compaction#557 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:29,314 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/916d551d5e1f476dae6a911e5dc76546 is 50, key is test_row_0/C:col10/1731527128025/Put/seqid=0 2024-11-13T19:45:29,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742464_1640 (size=13629) 2024-11-13T19:45:29,722 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/916d551d5e1f476dae6a911e5dc76546 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/916d551d5e1f476dae6a911e5dc76546 2024-11-13T19:45:29,726 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e12f682899aa55a4d182e84dc252f667/C of e12f682899aa55a4d182e84dc252f667 into 916d551d5e1f476dae6a911e5dc76546(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:29,726 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e12f682899aa55a4d182e84dc252f667: 2024-11-13T19:45:29,726 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., storeName=e12f682899aa55a4d182e84dc252f667/C, priority=12, startTime=1731527128878; duration=0sec 2024-11-13T19:45:29,726 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:29,726 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e12f682899aa55a4d182e84dc252f667:C 2024-11-13T19:45:30,195 INFO [Thread-2182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2182. 2024-11-13T19:45:30,195 DEBUG [Thread-2182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:30,195 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:30,195 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:30,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 980 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2937 rows 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2960 rows 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2930 rows 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2928 rows 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 985 2024-11-13T19:45:30,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2954 rows 2024-11-13T19:45:30,196 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:45:30,196 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:30,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:30,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:30,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:30,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:45:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:30,200 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527130200"}]},"ts":"1731527130200"} 2024-11-13T19:45:30,202 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:45:30,202 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:45:30,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:45:30,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, UNASSIGN}] 2024-11-13T19:45:30,205 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, UNASSIGN 2024-11-13T19:45:30,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e12f682899aa55a4d182e84dc252f667, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:30,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, UNASSIGN because future has completed 2024-11-13T19:45:30,207 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:45:30,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:30,360 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing e12f682899aa55a4d182e84dc252f667, disabling compactions & flushes 2024-11-13T19:45:30,360 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. after waiting 0 ms 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:30,360 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing e12f682899aa55a4d182e84dc252f667 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T19:45:30,360 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=A 2024-11-13T19:45:30,361 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:30,361 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=B 2024-11-13T19:45:30,361 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:30,361 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e12f682899aa55a4d182e84dc252f667, store=C 2024-11-13T19:45:30,361 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:30,364 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a324c6ac9b8047748daba727ac562cd8 is 50, key is test_row_0/A:col10/1731527130194/Put/seqid=0 2024-11-13T19:45:30,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742465_1641 (size=7415) 2024-11-13T19:45:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:30,767 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a324c6ac9b8047748daba727ac562cd8 2024-11-13T19:45:30,773 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3f3a3b2a20524f14a06904a558d6e006 is 50, key is test_row_0/B:col10/1731527130194/Put/seqid=0 2024-11-13T19:45:30,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742466_1642 (size=7415) 2024-11-13T19:45:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:31,177 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3f3a3b2a20524f14a06904a558d6e006 2024-11-13T19:45:31,184 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/38caf114d72a422bbd242fc48cdd0ba6 is 50, key is test_row_0/C:col10/1731527130194/Put/seqid=0 2024-11-13T19:45:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742467_1643 (size=7415) 2024-11-13T19:45:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:31,602 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/38caf114d72a422bbd242fc48cdd0ba6 2024-11-13T19:45:31,606 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/A/a324c6ac9b8047748daba727ac562cd8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a324c6ac9b8047748daba727ac562cd8 2024-11-13T19:45:31,608 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a324c6ac9b8047748daba727ac562cd8, entries=50, sequenceid=689, filesize=7.2 K 2024-11-13T19:45:31,609 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/B/3f3a3b2a20524f14a06904a558d6e006 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3f3a3b2a20524f14a06904a558d6e006 2024-11-13T19:45:31,611 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3f3a3b2a20524f14a06904a558d6e006, entries=50, sequenceid=689, filesize=7.2 K 2024-11-13T19:45:31,611 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/.tmp/C/38caf114d72a422bbd242fc48cdd0ba6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/38caf114d72a422bbd242fc48cdd0ba6 2024-11-13T19:45:31,614 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/38caf114d72a422bbd242fc48cdd0ba6, entries=50, sequenceid=689, filesize=7.2 K 2024-11-13T19:45:31,614 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for e12f682899aa55a4d182e84dc252f667 in 1254ms, sequenceid=689, compaction requested=false 2024-11-13T19:45:31,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/21a7ab68afb34ef2b268a715b2bc42fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db8945db5a214303b697f66fecdcfa84, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b131c45edaec4a3fb05891d171c982ee, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4b9a9b0981ea44cb85d97338530cad38, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5064b7749b9d41f882932ca957b988b8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4221c6cc3895455b87398e3c8195b736, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e4f448c9994342c1b7d8e7b37b5564a0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc9daf6a43e04a8898f2511b24aac20a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ee65e590bdf044c384eb77f049516522, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3808d6e69009487f8f3ab0a0d71efaf7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ecc3d0012d3a46a289dcb5f4aa7686bd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f] to archive 2024-11-13T19:45:31,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:31,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/21a7ab68afb34ef2b268a715b2bc42fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/21a7ab68afb34ef2b268a715b2bc42fa 2024-11-13T19:45:31,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5170eb2fa7c04cc6bbfe02f11ce69335 2024-11-13T19:45:31,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1fe786091096457993a8a8201566a571 2024-11-13T19:45:31,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/bcded46d2ef14e0ba66f0d69f9d3b82e 2024-11-13T19:45:31,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db8945db5a214303b697f66fecdcfa84 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db8945db5a214303b697f66fecdcfa84 2024-11-13T19:45:31,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/97e584829f404209b10c5a6dfb5da4e3 2024-11-13T19:45:31,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f000770d36a34a8699a6499cec8a7b0d 2024-11-13T19:45:31,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/1a1e08c8249840b48ded9daed332fd69 2024-11-13T19:45:31,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b131c45edaec4a3fb05891d171c982ee to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b131c45edaec4a3fb05891d171c982ee 2024-11-13T19:45:31,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5efa4d0b615f4663b1a8cb954ba35612 2024-11-13T19:45:31,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ed7af2a3936b414287ae8c5efae3e1cb 2024-11-13T19:45:31,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4b9a9b0981ea44cb85d97338530cad38 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4b9a9b0981ea44cb85d97338530cad38 2024-11-13T19:45:31,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e83c4c701db848748aa7f0ef94a3d163 2024-11-13T19:45:31,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/b8f592bfbc8a4d55b83da724e3b81584 2024-11-13T19:45:31,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5064b7749b9d41f882932ca957b988b8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/5064b7749b9d41f882932ca957b988b8 2024-11-13T19:45:31,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ab65f2519e6b4227a62be2561a485d1b 2024-11-13T19:45:31,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/30b58a8c9a14492483636f2e07004aa3 2024-11-13T19:45:31,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4221c6cc3895455b87398e3c8195b736 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/4221c6cc3895455b87398e3c8195b736 2024-11-13T19:45:31,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/86e4c2f8676846389dd89561dae714ab 2024-11-13T19:45:31,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/8b8699a71f21498ba40dc2a4b38d8306 2024-11-13T19:45:31,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e4f448c9994342c1b7d8e7b37b5564a0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e4f448c9994342c1b7d8e7b37b5564a0 2024-11-13T19:45:31,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/83a71291fc8047cdb66b41341ec5ae35 2024-11-13T19:45:31,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/f1dfc3bbdff04a21a69ef1663ff77776 2024-11-13T19:45:31,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a81dbf7c1ec04fc38a2d7eb7f43949bc 2024-11-13T19:45:31,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc9daf6a43e04a8898f2511b24aac20a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc9daf6a43e04a8898f2511b24aac20a 2024-11-13T19:45:31,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/190053d5537d4257a310e6df39eb5b07 2024-11-13T19:45:31,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/dc4ae120a7f74a91af94894247ba26e0 2024-11-13T19:45:31,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ee65e590bdf044c384eb77f049516522 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ee65e590bdf044c384eb77f049516522 2024-11-13T19:45:31,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/db5b4e3b7d8942febb8cc0f8cb0ff2b3 2024-11-13T19:45:31,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/314a4727fe9c47ef948531f83f23f828 2024-11-13T19:45:31,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3808d6e69009487f8f3ab0a0d71efaf7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/3808d6e69009487f8f3ab0a0d71efaf7 2024-11-13T19:45:31,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/23208ca51d4f414593657ca01d2a4a7e 2024-11-13T19:45:31,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ddf828fc7351440082c1e1110fc689d9 2024-11-13T19:45:31,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ecc3d0012d3a46a289dcb5f4aa7686bd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ecc3d0012d3a46a289dcb5f4aa7686bd 2024-11-13T19:45:31,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/aad375bfdd9040d89071b7f05fc74f3a 2024-11-13T19:45:31,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/e3ba1ba6cf8141fd804dbdecd616c9cd 2024-11-13T19:45:31,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/939ac56444d84e6289a565b035ad432f 2024-11-13T19:45:31,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/64962c83b6c24b309e72afffbba4fb2e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71b2dd9a445e4b81b857c82369b49f5e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/62acd14e4c5741f890584705e608dc6c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d1b97b55e1e04afe96a6d9a07848244d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7343ba4e83e14e90b5d834c72bfc52d9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d7d5f8b660d14965b649022875e30cc5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/145792158c044b62946c4b41d861ee43, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1ea11a64114949269136e02373dc3c34, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/9394d2ae95084ea183814f22752172e3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c0302234ea80468592ada845dfae65b2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fc1821e151244a5ba9cb6cab876768d0, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c] to archive 2024-11-13T19:45:31,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:31,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/64962c83b6c24b309e72afffbba4fb2e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/64962c83b6c24b309e72afffbba4fb2e 2024-11-13T19:45:31,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2cee5e749c8844f99be61c3f8f3ac44d 2024-11-13T19:45:31,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1b931b1d0e3142e7bdd57286354ca0d3 2024-11-13T19:45:31,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71b2dd9a445e4b81b857c82369b49f5e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71b2dd9a445e4b81b857c82369b49f5e 2024-11-13T19:45:31,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1f518946665049a197a8fb3e488b6e96 2024-11-13T19:45:31,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/db06653fafbe4985a546125c9ad0cac8 2024-11-13T19:45:31,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7470bd3a6e66486584b8196610f98a82 2024-11-13T19:45:31,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/62acd14e4c5741f890584705e608dc6c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/62acd14e4c5741f890584705e608dc6c 2024-11-13T19:45:31,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/e29ca054813e4ed7b398e0a24f720a1e 2024-11-13T19:45:31,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76a210d8f9b74066b39441c1c16006fa 2024-11-13T19:45:31,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d1b97b55e1e04afe96a6d9a07848244d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d1b97b55e1e04afe96a6d9a07848244d 2024-11-13T19:45:31,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1e70592063fe44ed8ef47d36a5ca6736 2024-11-13T19:45:31,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c4c139fb53e4bbb94f593ef430cb3ba 2024-11-13T19:45:31,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7343ba4e83e14e90b5d834c72bfc52d9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/7343ba4e83e14e90b5d834c72bfc52d9 2024-11-13T19:45:31,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c83ebf7802b54158b4a6a9004c693d66 2024-11-13T19:45:31,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3e98eebd6ea6467397c0238ccd6448b1 2024-11-13T19:45:31,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3cd8c3bfb1774f48b9e0744dbe829fb8 2024-11-13T19:45:31,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d7d5f8b660d14965b649022875e30cc5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d7d5f8b660d14965b649022875e30cc5 2024-11-13T19:45:31,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b4c787c1e605437098363e94fc22d78a 2024-11-13T19:45:31,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/ee8512f2a2e64c43a903974006519810 2024-11-13T19:45:31,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/145792158c044b62946c4b41d861ee43 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/145792158c044b62946c4b41d861ee43 2024-11-13T19:45:31,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/76dbc28e036a4427aead034e1405e2cd 2024-11-13T19:45:31,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/71357777b99c461ab79c582679c552b7 2024-11-13T19:45:31,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1ea11a64114949269136e02373dc3c34 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/1ea11a64114949269136e02373dc3c34 2024-11-13T19:45:31,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/d9a3ff04d7254d9db412237c2b53b56b 2024-11-13T19:45:31,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/01043690024e494293dddcc1f9bb11ab 2024-11-13T19:45:31,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/9394d2ae95084ea183814f22752172e3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/9394d2ae95084ea183814f22752172e3 2024-11-13T19:45:31,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f607f698d50b48da8f0c129b17776e91 2024-11-13T19:45:31,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f80e3dfba4d04497800092f8dc33007c 2024-11-13T19:45:31,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c0302234ea80468592ada845dfae65b2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/c0302234ea80468592ada845dfae65b2 2024-11-13T19:45:31,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/2bf6e21bb79247a586d8203a57482963 2024-11-13T19:45:31,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c716eed186e4313b04a42c362f2860f 2024-11-13T19:45:31,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fc1821e151244a5ba9cb6cab876768d0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/fc1821e151244a5ba9cb6cab876768d0 2024-11-13T19:45:31,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3463d63caa6b408db966636a1bcb00c3 2024-11-13T19:45:31,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/18f6dd7c1e03483fa18ee8740253cc5b 2024-11-13T19:45:31,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/5c3468e254324f20b68545acafa79b3f 2024-11-13T19:45:31,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/f71b36728c944162bf050e91a3bf096c 2024-11-13T19:45:31,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4dae4e6c4aa5428fbf20b06b56798be4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/fd9e797497cc44449ebc50d82999d143, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a4211c9d06204a27a53bdac15fb79673, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9a8d20d9c877433f94fb7668a2ac3ea6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/e54e3c5349cd4925b55e1828a23e13e2, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b9bfc05679764bc4a8d7309d5b45021a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/2a8dd99bb4ab488ca500349037438d6d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7612439f492047de88a8691475455250, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a684354fcc8b4ade81a5d12337e3d0f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4adad769f608469993a511db61284221, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154] to archive 2024-11-13T19:45:31,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:31,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4dae4e6c4aa5428fbf20b06b56798be4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4dae4e6c4aa5428fbf20b06b56798be4 2024-11-13T19:45:31,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7cd01d5efd9641be98188f176fe6b12c 2024-11-13T19:45:31,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/fd9e797497cc44449ebc50d82999d143 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/fd9e797497cc44449ebc50d82999d143 2024-11-13T19:45:31,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/5019439737a249d79d7cbfcabe1829c5 2024-11-13T19:45:31,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/05918dcc92f046b0b638f4434b840d6c 2024-11-13T19:45:31,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/8469bffa8f6846dab75c75b207d81bd5 2024-11-13T19:45:31,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a4211c9d06204a27a53bdac15fb79673 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a4211c9d06204a27a53bdac15fb79673 2024-11-13T19:45:31,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/372f09fac8b0425ebebf381bc4404ec1 2024-11-13T19:45:31,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9ea57c7f10da4bbe85a0cc72688a6169 2024-11-13T19:45:31,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/efcec0b15abf40aeb1b2ec7e5c98ce9d 2024-11-13T19:45:31,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9a8d20d9c877433f94fb7668a2ac3ea6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9a8d20d9c877433f94fb7668a2ac3ea6 2024-11-13T19:45:31,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/383b830eaac04ddca7dd7664d142f13e 2024-11-13T19:45:31,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/73763b2318d44096833c3aeb52c8a677 2024-11-13T19:45:31,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/1c3d7be09627423db164dbccfc390c5f 2024-11-13T19:45:31,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f616276d2ec74cbabb76052e02b38f61 2024-11-13T19:45:31,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4fe5b49fe2cd4b3d9c49f54dc56f7e14 2024-11-13T19:45:31,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/04e1b9b7393644e98edca711c6c777ed 2024-11-13T19:45:31,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f84a4801194f480d82cab43902086198 2024-11-13T19:45:31,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/dfcc2f7fc1e149e4a30e89c49a2d145d 2024-11-13T19:45:31,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/e54e3c5349cd4925b55e1828a23e13e2 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/e54e3c5349cd4925b55e1828a23e13e2 2024-11-13T19:45:31,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/721025085143407280b0ed50a34447fc 2024-11-13T19:45:31,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/94836c538a794f2cb336686b94f6b2e9 2024-11-13T19:45:31,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b9bfc05679764bc4a8d7309d5b45021a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b9bfc05679764bc4a8d7309d5b45021a 2024-11-13T19:45:31,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7e88459b21f94b23b81c51bd331e89b4 2024-11-13T19:45:31,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/b27b935bfcdf450bb12d08506cb08bb8 2024-11-13T19:45:31,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/2a8dd99bb4ab488ca500349037438d6d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/2a8dd99bb4ab488ca500349037438d6d 2024-11-13T19:45:31,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a65ee6b162c44f48af89180e40a68b81 2024-11-13T19:45:31,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/59ea7c05da48481ca5eea42eb562c95f 2024-11-13T19:45:31,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7612439f492047de88a8691475455250 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/7612439f492047de88a8691475455250 2024-11-13T19:45:31,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/004d2871c34f452cbc8693875a514deb 2024-11-13T19:45:31,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/0defb7658d834c33b08d589b4ea1d90e 2024-11-13T19:45:31,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a684354fcc8b4ade81a5d12337e3d0f6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a684354fcc8b4ade81a5d12337e3d0f6 2024-11-13T19:45:31,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/a9f7ea5e27934ce1a808b94ecc2fb627 2024-11-13T19:45:31,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/115cb319d21e4256bf685a1592ec8148 2024-11-13T19:45:31,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4adad769f608469993a511db61284221 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4adad769f608469993a511db61284221 2024-11-13T19:45:31,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/9743b13b09ca428a80765079cc4bc3e5 2024-11-13T19:45:31,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/f0a69d26981a40a38ccf60d0d36b6d1c 2024-11-13T19:45:31,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/4b3cbf2a8e6c4233a1f8f92dac84cf57 2024-11-13T19:45:31,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/53cd8a7d4b784b4c8ea10ba4a61f6154 2024-11-13T19:45:31,717 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/recovered.edits/692.seqid, newMaxSeqId=692, maxSeqId=1 2024-11-13T19:45:31,718 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. 2024-11-13T19:45:31,718 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for e12f682899aa55a4d182e84dc252f667: Waiting for close lock at 1731527130360Running coprocessor pre-close hooks at 1731527130360Disabling compacts and flushes for region at 1731527130360Disabling writes for close at 1731527130360Obtaining lock to block concurrent updates at 1731527130360Preparing flush snapshotting stores in e12f682899aa55a4d182e84dc252f667 at 1731527130360Finished memstore snapshotting TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731527130361 (+1 ms)Flushing stores of TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667. at 1731527130361Flushing e12f682899aa55a4d182e84dc252f667/A: creating writer at 1731527130361Flushing e12f682899aa55a4d182e84dc252f667/A: appending metadata at 1731527130363 (+2 ms)Flushing e12f682899aa55a4d182e84dc252f667/A: closing flushed file at 1731527130363Flushing e12f682899aa55a4d182e84dc252f667/B: creating writer at 1731527130771 (+408 ms)Flushing e12f682899aa55a4d182e84dc252f667/B: appending metadata at 1731527130772 (+1 ms)Flushing e12f682899aa55a4d182e84dc252f667/B: closing flushed file at 1731527130772Flushing e12f682899aa55a4d182e84dc252f667/C: creating writer at 1731527131181 (+409 ms)Flushing e12f682899aa55a4d182e84dc252f667/C: appending metadata at 1731527131183 (+2 ms)Flushing e12f682899aa55a4d182e84dc252f667/C: closing flushed file at 1731527131183Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25460a0a: reopening flushed file at 1731527131605 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@425eeed1: reopening flushed file at 1731527131608 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e1e820: reopening flushed file at 1731527131611 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for e12f682899aa55a4d182e84dc252f667 in 1254ms, sequenceid=689, compaction requested=false at 1731527131614 (+3 ms)Writing region close event to WAL at 1731527131715 (+101 ms)Running coprocessor post-close hooks at 1731527131718 (+3 ms)Closed at 1731527131718 2024-11-13T19:45:31,720 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:31,720 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e12f682899aa55a4d182e84dc252f667, regionState=CLOSED 2024-11-13T19:45:31,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:31,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T19:45:31,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure e12f682899aa55a4d182e84dc252f667, server=54d0784d1943,36227,1731526989835 in 1.5150 sec 2024-11-13T19:45:31,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-13T19:45:31,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e12f682899aa55a4d182e84dc252f667, UNASSIGN in 1.5200 sec 2024-11-13T19:45:31,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T19:45:31,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5230 sec 2024-11-13T19:45:31,729 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527131729"}]},"ts":"1731527131729"} 2024-11-13T19:45:31,730 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:45:31,730 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:45:31,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5350 sec 2024-11-13T19:45:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:45:32,344 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:32,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:45:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,347 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T19:45:32,348 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,359 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:32,361 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/recovered.edits] 2024-11-13T19:45:32,363 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a324c6ac9b8047748daba727ac562cd8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/a324c6ac9b8047748daba727ac562cd8 2024-11-13T19:45:32,365 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ec025ffd267e496e82c21d3866dcd610 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/A/ec025ffd267e496e82c21d3866dcd610 2024-11-13T19:45:32,367 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3f3a3b2a20524f14a06904a558d6e006 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/3f3a3b2a20524f14a06904a558d6e006 2024-11-13T19:45:32,372 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b18bc2d241c045358ea0759ea37891a9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/B/b18bc2d241c045358ea0759ea37891a9 2024-11-13T19:45:32,379 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/38caf114d72a422bbd242fc48cdd0ba6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/38caf114d72a422bbd242fc48cdd0ba6 2024-11-13T19:45:32,380 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/916d551d5e1f476dae6a911e5dc76546 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/C/916d551d5e1f476dae6a911e5dc76546 2024-11-13T19:45:32,383 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/recovered.edits/692.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667/recovered.edits/692.seqid 2024-11-13T19:45:32,384 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/e12f682899aa55a4d182e84dc252f667 2024-11-13T19:45:32,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:45:32,386 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,389 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:45:32,402 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:45:32,410 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,410 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:45:32,411 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527132410"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:32,413 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:45:32,413 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e12f682899aa55a4d182e84dc252f667, NAME => 'TestAcidGuarantees,,1731527104660.e12f682899aa55a4d182e84dc252f667.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:45:32,413 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:45:32,413 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527132413"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:32,416 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:45:32,416 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-13T19:45:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T19:45:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:45:32,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:32,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 252), OpenFileDescriptor=455 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=776 (was 804), ProcessCount=11 (was 11), AvailableMemoryMB=1508 (was 261) - AvailableMemoryMB LEAK? - 2024-11-13T19:45:32,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=776, ProcessCount=11, AvailableMemoryMB=1503 2024-11-13T19:45:32,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:45:32,486 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:45:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:32,489 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:45:32,489 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:32,489 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-13T19:45:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:45:32,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:45:32,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742468_1644 (size=1061) 2024-11-13T19:45:32,509 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e 2024-11-13T19:45:32,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742469_1645 (size=53) 2024-11-13T19:45:32,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:45:32,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f989ae1f218e9e3abedc1d7da78f0f53, disabling compactions & flushes 2024-11-13T19:45:32,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. after waiting 0 ms 2024-11-13T19:45:32,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f989ae1f218e9e3abedc1d7da78f0f53: Waiting for close lock at 1731527132529Disabling compacts and flushes for region at 1731527132529Disabling writes for close at 1731527132529Writing region close event to WAL at 1731527132529Closed at 1731527132529 2024-11-13T19:45:32,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:45:32,531 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731527132531"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527132531"}]},"ts":"1731527132531"} 2024-11-13T19:45:32,532 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:45:32,533 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:45:32,534 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527132533"}]},"ts":"1731527132533"} 2024-11-13T19:45:32,535 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:45:32,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, ASSIGN}] 2024-11-13T19:45:32,536 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, ASSIGN 2024-11-13T19:45:32,537 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, ASSIGN; state=OFFLINE, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=false 2024-11-13T19:45:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:45:32,688 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:32,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, ASSIGN because future has completed 2024-11-13T19:45:32,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:45:32,846 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,846 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:45:32,847 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,847 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:45:32,847 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,847 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,848 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,849 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:32,849 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName A 2024-11-13T19:45:32,850 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:32,850 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:32,850 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,851 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:32,851 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName B 2024-11-13T19:45:32,852 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:32,852 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:32,852 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,853 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:32,853 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName C 2024-11-13T19:45:32,853 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:32,856 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:32,856 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,856 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,857 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,857 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,860 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,860 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,861 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:45:32,862 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,865 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:45:32,865 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened f989ae1f218e9e3abedc1d7da78f0f53; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72779180, jitterRate=0.08449429273605347}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:45:32,866 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:32,866 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for f989ae1f218e9e3abedc1d7da78f0f53: Running coprocessor pre-open hook at 1731527132847Writing region info on filesystem at 1731527132847Initializing all the Stores at 1731527132847Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527132847Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527132848 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527132848Cleaning up temporary data from old regions at 1731527132860 (+12 ms)Running coprocessor post-open hooks at 1731527132866 (+6 ms)Region opened successfully at 1731527132866 2024-11-13T19:45:32,867 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., pid=140, masterSystemTime=1731527132843 2024-11-13T19:45:32,869 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,869 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:32,870 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:32,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:32,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T19:45:32,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 in 182 msec 2024-11-13T19:45:32,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T19:45:32,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, ASSIGN in 338 msec 2024-11-13T19:45:32,877 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:45:32,877 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527132877"}]},"ts":"1731527132877"} 2024-11-13T19:45:32,878 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:45:32,879 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:45:32,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 393 msec 2024-11-13T19:45:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:45:33,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:33,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f7dbdf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,117 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd11ee2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,118 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541dcae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,119 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,122 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:45:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,42037,1731526988795 2024-11-13T19:45:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1976218f 2024-11-13T19:45:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:45:33,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:45:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:45:33,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:45:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742470_1646 (size=1097) 2024-11-13T19:45:33,141 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T19:45:33,141 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T19:45:33,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:45:33,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE}] 2024-11-13T19:45:33,147 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE 2024-11-13T19:45:33,148 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE because future has completed 2024-11-13T19:45:33,150 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:45:33,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:33,302 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,302 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:45:33,302 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing f989ae1f218e9e3abedc1d7da78f0f53, disabling compactions & flushes 2024-11-13T19:45:33,302 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,302 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,302 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. after waiting 0 ms 2024-11-13T19:45:33,302 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,306 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:45:33,306 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,306 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for f989ae1f218e9e3abedc1d7da78f0f53: Waiting for close lock at 1731527133302Running coprocessor pre-close hooks at 1731527133302Disabling compacts and flushes for region at 1731527133302Disabling writes for close at 1731527133302Writing region close event to WAL at 1731527133303 (+1 ms)Running coprocessor post-close hooks at 1731527133306 (+3 ms)Closed at 1731527133306 2024-11-13T19:45:33,306 WARN [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: f989ae1f218e9e3abedc1d7da78f0f53 to self. 2024-11-13T19:45:33,308 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=CLOSED 2024-11-13T19:45:33,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:33,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T19:45:33,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 in 161 msec 2024-11-13T19:45:33,313 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE; state=CLOSED, location=54d0784d1943,36227,1731526989835; forceNewPlan=false, retain=true 2024-11-13T19:45:33,464 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=OPENING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE because future has completed 2024-11-13T19:45:33,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:33,620 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,620 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:45:33,620 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,620 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:45:33,620 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,620 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,622 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,622 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:33,622 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName A 2024-11-13T19:45:33,623 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:33,624 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:33,624 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,624 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:33,625 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName B 2024-11-13T19:45:33,625 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:33,625 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:33,625 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,626 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T19:45:33,626 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f989ae1f218e9e3abedc1d7da78f0f53 columnFamilyName C 2024-11-13T19:45:33,626 DEBUG [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:33,626 INFO [StoreOpener-f989ae1f218e9e3abedc1d7da78f0f53-1 {}] regionserver.HStore(327): Store=f989ae1f218e9e3abedc1d7da78f0f53/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:45:33,626 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,626 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,627 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,627 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,628 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,628 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,628 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:45:33,629 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,630 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened f989ae1f218e9e3abedc1d7da78f0f53; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73575207, jitterRate=0.09635601937770844}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:45:33,630 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,630 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for f989ae1f218e9e3abedc1d7da78f0f53: Running coprocessor pre-open hook at 1731527133621Writing region info on filesystem at 1731527133621Initializing all the Stores at 1731527133621Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527133621Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527133621Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527133621Cleaning up temporary data from old regions at 1731527133628 (+7 ms)Running coprocessor post-open hooks at 1731527133630 (+2 ms)Region opened successfully at 1731527133630 2024-11-13T19:45:33,631 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., pid=145, masterSystemTime=1731527133617 2024-11-13T19:45:33,632 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,632 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,633 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=OPEN, openSeqNum=5, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:33,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-13T19:45:33,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 in 169 msec 2024-11-13T19:45:33,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T19:45:33,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, REOPEN/MOVE in 490 msec 2024-11-13T19:45:33,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T19:45:33,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-13T19:45:33,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 513 msec 2024-11-13T19:45:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T19:45:33,641 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:33,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a5d471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779fb745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592506c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7663d9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10b338e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ece8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876a305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aeee231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6b1715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ddb59f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3229edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79538ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62aa2b5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65be243a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32aa3c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d71a27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b0e8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d16a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bce89c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1833782d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16487325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d06b23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42048694, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5129a55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c057643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc2cf16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f0ee4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,42037,-1 for getting cluster id 2024-11-13T19:45:33,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:45:33,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abd38298-c95a-46fd-865b-a2bfc93c7871' 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abd38298-c95a-46fd-865b-a2bfc93c7871" 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e23459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,42037,-1] 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:45:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:33,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:45:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b6fc03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:45:33,679 DEBUG [Thread-2833 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,679 DEBUG [Thread-2829 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,680 DEBUG [Thread-2827 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,680 DEBUG [Thread-2846 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,680 DEBUG [Thread-2838 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,680 DEBUG [Thread-2844 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,681 DEBUG [Thread-2840 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,682 DEBUG [Thread-2835 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,683 DEBUG [Thread-2831 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,683 DEBUG [Thread-2842 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:45:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,684 DEBUG [Thread-2831 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,684 DEBUG [Thread-2831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:33,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,36227,1731526989835, seqNum=-1] 2024-11-13T19:45:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:45:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:45:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-13T19:45:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:33,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:33,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5] 2024-11-13T19:45:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:33,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:33,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527133734, max timestamp=1731527133747Num uniques 859; 2024-11-13T19:45:33,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:33,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527133734, max timestamp=1731527133747Num uniques 859; 2024-11-13T19:45:33,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:33,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527133734, max timestamp=1731527133747Num uniques 859; 2024-11-13T19:45:33,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:33,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113473d5682d5be4a249412f0306b638b39_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527133698/Put/seqid=0 2024-11-13T19:45:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527193785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527193786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527193789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527193791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527193791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742471_1647 (size=12154) 2024-11-13T19:45:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:33,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:33,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:33,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527193894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527193904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527193905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527193906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527193905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527194104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527194114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527194114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527194120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527194120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:34,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113473d5682d5be4a249412f0306b638b39_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113473d5682d5be4a249412f0306b638b39_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:34,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/76b6130cb5db47cfa1308e3ff7b99f17, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:34,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/76b6130cb5db47cfa1308e3ff7b99f17 is 175, key is test_row_0/A:col10/1731527133698/Put/seqid=0 2024-11-13T19:45:34,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742472_1648 (size=30955) 2024-11-13T19:45:34,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527194417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527194424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527194425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527194428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527194435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/76b6130cb5db47cfa1308e3ff7b99f17 2024-11-13T19:45:34,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/76c0b8f1dbb449f885472ad80cde945d is 50, key is test_row_0/B:col10/1731527133698/Put/seqid=0 2024-11-13T19:45:34,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742473_1649 (size=12001) 2024-11-13T19:45:34,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/76c0b8f1dbb449f885472ad80cde945d 2024-11-13T19:45:34,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/3eb9674b711841e28d8e5201d1e806b4 is 50, key is test_row_0/C:col10/1731527133698/Put/seqid=0 2024-11-13T19:45:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742474_1650 (size=12001) 2024-11-13T19:45:34,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/3eb9674b711841e28d8e5201d1e806b4 2024-11-13T19:45:34,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/76b6130cb5db47cfa1308e3ff7b99f17 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17 2024-11-13T19:45:34,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T19:45:34,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/76c0b8f1dbb449f885472ad80cde945d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d 2024-11-13T19:45:34,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:45:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/3eb9674b711841e28d8e5201d1e806b4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4 2024-11-13T19:45:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:34,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:45:34,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f989ae1f218e9e3abedc1d7da78f0f53 in 1141ms, sequenceid=16, compaction requested=false 2024-11-13T19:45:34,845 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:45:34,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:34,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T19:45:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527194946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:34,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527194947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:34,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527194949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527194950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527194954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:34,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113605de99a26b047379dcc1bf2b74766ca_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527134935/Put/seqid=0 2024-11-13T19:45:34,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742475_1651 (size=12154) 2024-11-13T19:45:34,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:35,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113605de99a26b047379dcc1bf2b74766ca_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113605de99a26b047379dcc1bf2b74766ca_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:35,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c3f870949570413f80629bccf23defb8, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:35,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c3f870949570413f80629bccf23defb8 is 175, key is test_row_0/A:col10/1731527134935/Put/seqid=0 2024-11-13T19:45:35,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742476_1652 (size=30955) 2024-11-13T19:45:35,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c3f870949570413f80629bccf23defb8 2024-11-13T19:45:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527195053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527195053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527195064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527195064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/15277fd8250b443d8646a35b83def81c is 50, key is test_row_0/B:col10/1731527134935/Put/seqid=0 2024-11-13T19:45:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742477_1653 (size=12001) 2024-11-13T19:45:35,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:35,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:35,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527195264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527195265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527195274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527195275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:35,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:35,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/15277fd8250b443d8646a35b83def81c 2024-11-13T19:45:35,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/ac794e60faea495098960e56c972f5b1 is 50, key is test_row_0/C:col10/1731527134935/Put/seqid=0 2024-11-13T19:45:35,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:35,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527195574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527195574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742478_1654 (size=12001) 2024-11-13T19:45:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527195584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/ac794e60faea495098960e56c972f5b1 2024-11-13T19:45:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527195594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:35,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c3f870949570413f80629bccf23defb8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8 2024-11-13T19:45:35,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8, entries=150, sequenceid=53, filesize=30.2 K 2024-11-13T19:45:35,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/15277fd8250b443d8646a35b83def81c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c 2024-11-13T19:45:35,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c, entries=150, sequenceid=53, filesize=11.7 K 2024-11-13T19:45:35,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/ac794e60faea495098960e56c972f5b1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1 2024-11-13T19:45:35,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1, entries=150, sequenceid=53, filesize=11.7 K 2024-11-13T19:45:35,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f989ae1f218e9e3abedc1d7da78f0f53 in 748ms, sequenceid=53, compaction requested=false 2024-11-13T19:45:35,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:35,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131003f75662914932a28a3babd331fbcc_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527134943/Put/seqid=0 2024-11-13T19:45:35,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742479_1655 (size=12154) 2024-11-13T19:45:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:35,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:36,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527136053, max timestamp=1731527136079Num uniques 859; 2024-11-13T19:45:36,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:36,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527136053, max timestamp=1731527136079Num uniques 859; 2024-11-13T19:45:36,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527136053, max timestamp=1731527136079Num uniques 859; 2024-11-13T19:45:36,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:36,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527136102, max timestamp=1731527136125Num uniques 1018; 2024-11-13T19:45:36,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-13T19:45:36,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731527136102, max timestamp=1731527136129Num uniques 1018; 2024-11-13T19:45:36,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:45:36,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9971, totalHeapSize=22912, min timestamp=1731527136102, max timestamp=1731527136129Num uniques 1018; 2024-11-13T19:45:36,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46611, new segment data size=9971, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T19:45:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527196160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527196162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527196164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527196165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527196172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:36,203 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131003f75662914932a28a3babd331fbcc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131003f75662914932a28a3babd331fbcc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/3db6c5339bd14bf4b134f658f6ac0495, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/3db6c5339bd14bf4b134f658f6ac0495 is 175, key is test_row_0/A:col10/1731527134943/Put/seqid=0 2024-11-13T19:45:36,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742480_1656 (size=30955) 2024-11-13T19:45:36,244 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/3db6c5339bd14bf4b134f658f6ac0495 2024-11-13T19:45:36,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/38afc8c5b1e8466ca11661f79b187577 is 50, key is test_row_0/B:col10/1731527134943/Put/seqid=0 2024-11-13T19:45:36,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742481_1657 (size=12001) 2024-11-13T19:45:36,277 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/38afc8c5b1e8466ca11661f79b187577 2024-11-13T19:45:36,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527196275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/9bf8859ebf7d4959910ab6046af1aca7 is 50, key is test_row_0/C:col10/1731527134943/Put/seqid=0 2024-11-13T19:45:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527196286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527196286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527196286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527196287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742482_1658 (size=12001) 2024-11-13T19:45:36,295 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/9bf8859ebf7d4959910ab6046af1aca7 2024-11-13T19:45:36,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/3db6c5339bd14bf4b134f658f6ac0495 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495 2024-11-13T19:45:36,304 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495, entries=150, sequenceid=62, filesize=30.2 K 2024-11-13T19:45:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/38afc8c5b1e8466ca11661f79b187577 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577 2024-11-13T19:45:36,310 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T19:45:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/9bf8859ebf7d4959910ab6046af1aca7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7 2024-11-13T19:45:36,314 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7, entries=150, sequenceid=62, filesize=11.7 K 2024-11-13T19:45:36,315 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=143.13 KB/146560 for f989ae1f218e9e3abedc1d7da78f0f53 in 588ms, sequenceid=62, compaction requested=true 2024-11-13T19:45:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-13T19:45:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-13T19:45:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T19:45:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-11-13T19:45:36,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6430 sec 2024-11-13T19:45:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527136160, max timestamp=1731527136484Num uniques 1077; 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527136160, max timestamp=1731527136484Num uniques 1127; 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527136160, max timestamp=1731527136484Num uniques 1077; 2024-11-13T19:45:36,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-13T19:45:36,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=26.84 KB heapSize=64.55 KB 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376b0eba736ee47fe94be0f9308216770_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742483_1659 (size=14594) 2024-11-13T19:45:36,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:36,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376b0eba736ee47fe94be0f9308216770_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376b0eba736ee47fe94be0f9308216770_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:36,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6f556f95a3844c65b2b31cfcf0580085, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6f556f95a3844c65b2b31cfcf0580085 is 175, key is test_row_0/A:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:36,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742484_1660 (size=39549) 2024-11-13T19:45:36,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6f556f95a3844c65b2b31cfcf0580085 2024-11-13T19:45:36,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/99af8857a7a74fc39d34988a8f2bf03b is 50, key is test_row_0/B:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742485_1661 (size=12001) 2024-11-13T19:45:36,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/99af8857a7a74fc39d34988a8f2bf03b 2024-11-13T19:45:36,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527136518, max timestamp=1731527136528Num uniques 859; 2024-11-13T19:45:36,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:45:36,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731527136518, max timestamp=1731527136528Num uniques 859; 2024-11-13T19:45:36,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T19:45:36,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527136520, max timestamp=1731527136528Num uniques 859; 2024-11-13T19:45:36,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:36,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a81d5b41e4484998a65a842e9e04dc5d is 50, key is test_row_0/C:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742486_1662 (size=12001) 2024-11-13T19:45:36,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a81d5b41e4484998a65a842e9e04dc5d 2024-11-13T19:45:36,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6f556f95a3844c65b2b31cfcf0580085 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085 2024-11-13T19:45:36,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085, entries=200, sequenceid=117, filesize=38.6 K 2024-11-13T19:45:36,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/99af8857a7a74fc39d34988a8f2bf03b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b 2024-11-13T19:45:36,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527136566, max timestamp=1731527136577Num uniques 1118; 2024-11-13T19:45:36,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731527136566, max timestamp=1731527136577Num uniques 1118; 2024-11-13T19:45:36,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-13T19:45:36,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-13T19:45:36,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731527136566, max timestamp=1731527136577Num uniques 1018; 2024-11-13T19:45:36,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-13T19:45:36,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T19:45:36,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a81d5b41e4484998a65a842e9e04dc5d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d 2024-11-13T19:45:36,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T19:45:36,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~64.50 KB/66048, currentSize=55.11 KB/56432 for f989ae1f218e9e3abedc1d7da78f0f53 in 105ms, sequenceid=117, compaction requested=true 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:36,595 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:36,595 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:36,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:36,602 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:36,602 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:36,602 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:36,602 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=46.9 K 2024-11-13T19:45:36,602 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:36,602 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:36,602 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:36,603 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=129.3 K 2024-11-13T19:45:36,603 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:36,603 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085] 2024-11-13T19:45:36,603 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c0b8f1dbb449f885472ad80cde945d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527133696 2024-11-13T19:45:36,603 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b6130cb5db47cfa1308e3ff7b99f17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527133696 2024-11-13T19:45:36,603 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 15277fd8250b443d8646a35b83def81c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731527133789 2024-11-13T19:45:36,604 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 38afc8c5b1e8466ca11661f79b187577, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731527134943 2024-11-13T19:45:36,604 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f870949570413f80629bccf23defb8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731527133789 2024-11-13T19:45:36,604 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 99af8857a7a74fc39d34988a8f2bf03b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:36,604 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3db6c5339bd14bf4b134f658f6ac0495, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731527134943 2024-11-13T19:45:36,604 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f556f95a3844c65b2b31cfcf0580085, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:36,616 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,620 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#574 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:36,620 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/65f92892d4ae4a4c8d2f89455561a4ce is 50, key is test_row_0/B:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:36,621 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113e4e60ba18c884102b6bec4e74a024f23_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,624 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113e4e60ba18c884102b6bec4e74a024f23_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,624 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e4e60ba18c884102b6bec4e74a024f23_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:36,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742487_1663 (size=12139) 2024-11-13T19:45:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:36,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=61.82 KB heapSize=163.56 KB 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:36,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:36,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742488_1664 (size=4469) 2024-11-13T19:45:36,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134a6d0f2fd73e4fc3b6d83fbe2a41dd05_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527136597/Put/seqid=0 2024-11-13T19:45:36,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742489_1665 (size=12304) 2024-11-13T19:45:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527196662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527196668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527196668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527196672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527196673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527196785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527196794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527196794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527196794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:36,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527196794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527197004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527197004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527197004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527197004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527197013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,031 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/65f92892d4ae4a4c8d2f89455561a4ce as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/65f92892d4ae4a4c8d2f89455561a4ce 2024-11-13T19:45:37,036 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#573 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:37,036 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/de894cb7cd114fb9bb4bee277dac89f5 is 175, key is test_row_0/A:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:37,039 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 65f92892d4ae4a4c8d2f89455561a4ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:37,039 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:37,039 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=12, startTime=1731527136595; duration=0sec 2024-11-13T19:45:37,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:37,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:37,040 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:37,041 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:37,041 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:37,041 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:37,042 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=46.9 K 2024-11-13T19:45:37,043 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb9674b711841e28d8e5201d1e806b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731527133696 2024-11-13T19:45:37,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742490_1666 (size=31093) 2024-11-13T19:45:37,043 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting ac794e60faea495098960e56c972f5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731527133789 2024-11-13T19:45:37,044 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf8859ebf7d4959910ab6046af1aca7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731527134943 2024-11-13T19:45:37,044 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a81d5b41e4484998a65a842e9e04dc5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:37,060 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#576 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:37,060 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/574e562bcb5b461d8a9cb557cb70aa46 is 50, key is test_row_0/C:col10/1731527136484/Put/seqid=0 2024-11-13T19:45:37,061 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/de894cb7cd114fb9bb4bee277dac89f5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5 2024-11-13T19:45:37,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:37,066 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into de894cb7cd114fb9bb4bee277dac89f5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:37,066 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:37,066 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=12, startTime=1731527136595; duration=0sec 2024-11-13T19:45:37,067 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:37,067 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:37,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742491_1667 (size=12139) 2024-11-13T19:45:37,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134a6d0f2fd73e4fc3b6d83fbe2a41dd05_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6d0f2fd73e4fc3b6d83fbe2a41dd05_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:37,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c69115c771d44738be8d0d4f2a328ff6, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:37,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c69115c771d44738be8d0d4f2a328ff6 is 175, key is test_row_0/A:col10/1731527136597/Put/seqid=0 2024-11-13T19:45:37,075 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/574e562bcb5b461d8a9cb557cb70aa46 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/574e562bcb5b461d8a9cb557cb70aa46 2024-11-13T19:45:37,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742492_1668 (size=31105) 2024-11-13T19:45:37,079 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 574e562bcb5b461d8a9cb557cb70aa46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:37,079 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:37,079 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=12, startTime=1731527136595; duration=0sec 2024-11-13T19:45:37,079 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:37,079 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:37,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c69115c771d44738be8d0d4f2a328ff6 2024-11-13T19:45:37,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/0612bbb4b31344d79bfcbe1c94096fa1 is 50, key is test_row_0/B:col10/1731527136597/Put/seqid=0 2024-11-13T19:45:37,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742493_1669 (size=12151) 2024-11-13T19:45:37,255 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:45:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527197313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527197314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527197314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527197314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527197324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/0612bbb4b31344d79bfcbe1c94096fa1 2024-11-13T19:45:37,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/34a3c9b00b7f4999a23f5ad9982ef444 is 50, key is test_row_0/C:col10/1731527136597/Put/seqid=0 2024-11-13T19:45:37,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742494_1670 (size=12151) 2024-11-13T19:45:37,707 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:45:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527197823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527197823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527197823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527197824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527197833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:45:37,864 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:37,865 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-13T19:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:45:37,868 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:37,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:37,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:37,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.56 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/34a3c9b00b7f4999a23f5ad9982ef444 2024-11-13T19:45:37,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/c69115c771d44738be8d0d4f2a328ff6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6 2024-11-13T19:45:37,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6, entries=150, sequenceid=161, filesize=30.4 K 2024-11-13T19:45:37,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/0612bbb4b31344d79bfcbe1c94096fa1 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1 2024-11-13T19:45:37,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T19:45:37,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/34a3c9b00b7f4999a23f5ad9982ef444 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444 2024-11-13T19:45:37,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T19:45:37,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~75.24 KB/77042, heapSize ~203.36 KB/208240, currentSize=107.34 KB/109920 for f989ae1f218e9e3abedc1d7da78f0f53 in 1288ms, sequenceid=161, compaction requested=false 2024-11-13T19:45:37,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:45:38,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T19:45:38,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:38,028 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:38,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111329008eed2777456895388b681da83a59_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527136662/Put/seqid=0 2024-11-13T19:45:38,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742495_1671 (size=12304) 2024-11-13T19:45:38,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:38,071 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111329008eed2777456895388b681da83a59_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329008eed2777456895388b681da83a59_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/584835d77e804d3e8a73afea3eed4957, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/584835d77e804d3e8a73afea3eed4957 is 175, key is test_row_0/A:col10/1731527136662/Put/seqid=0 2024-11-13T19:45:38,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742496_1672 (size=31105) 2024-11-13T19:45:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:45:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:45:38,524 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/584835d77e804d3e8a73afea3eed4957 2024-11-13T19:45:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/e933d937735743ca885268bd354d936d is 50, key is test_row_0/B:col10/1731527136662/Put/seqid=0 2024-11-13T19:45:38,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742497_1673 (size=12151) 2024-11-13T19:45:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527198844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527198846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527198850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527198851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527198851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,948 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/e933d937735743ca885268bd354d936d 2024-11-13T19:45:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/91258d0149934acb8fce16f3ead67d05 is 50, key is test_row_0/C:col10/1731527136662/Put/seqid=0 2024-11-13T19:45:38,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527198953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742498_1674 (size=12151) 2024-11-13T19:45:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,959 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/91258d0149934acb8fce16f3ead67d05 2024-11-13T19:45:38,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/584835d77e804d3e8a73afea3eed4957 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957 2024-11-13T19:45:38,966 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957, entries=150, sequenceid=183, filesize=30.4 K 2024-11-13T19:45:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527198964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527198964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/e933d937735743ca885268bd354d936d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d 2024-11-13T19:45:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527198964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527198964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:38,971 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d, entries=150, sequenceid=183, filesize=11.9 K 2024-11-13T19:45:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/91258d0149934acb8fce16f3ead67d05 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05 2024-11-13T19:45:38,975 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05, entries=150, sequenceid=183, filesize=11.9 K 2024-11-13T19:45:38,976 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f989ae1f218e9e3abedc1d7da78f0f53 in 948ms, sequenceid=183, compaction requested=true 2024-11-13T19:45:38,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:38,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:38,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-13T19:45:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-13T19:45:38,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T19:45:38,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-13T19:45:38,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.1140 sec 2024-11-13T19:45:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:45:39,004 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:39,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-13T19:45:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:45:39,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:39,008 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:39,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:45:39,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T19:45:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:39,162 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:39,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d8c878333d3f4998a43d6012810252bc_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527138849/Put/seqid=0 2024-11-13T19:45:39,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742499_1675 (size=12304) 2024-11-13T19:45:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527199190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527199190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527199191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527199195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527199196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527199304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527199304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527199305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527199305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527199314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:45:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527199514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527199515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527199517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527199518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527199523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:39,598 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d8c878333d3f4998a43d6012810252bc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8c878333d3f4998a43d6012810252bc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dafbf1b665e442ef8d91c814c00640f4, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dafbf1b665e442ef8d91c814c00640f4 is 175, key is test_row_0/A:col10/1731527138849/Put/seqid=0 2024-11-13T19:45:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742500_1676 (size=31105) 2024-11-13T19:45:39,632 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dafbf1b665e442ef8d91c814c00640f4 2024-11-13T19:45:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:45:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/01e62a4358604b649cfba5e981dfe35a is 50, key is test_row_0/B:col10/1731527138849/Put/seqid=0 2024-11-13T19:45:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742501_1677 (size=12151) 2024-11-13T19:45:39,664 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/01e62a4358604b649cfba5e981dfe35a 2024-11-13T19:45:39,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/81d6bbf479714c8e9461e5588ffd8a47 is 50, key is test_row_0/C:col10/1731527138849/Put/seqid=0 2024-11-13T19:45:39,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742502_1678 (size=12151) 2024-11-13T19:45:39,740 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/81d6bbf479714c8e9461e5588ffd8a47 2024-11-13T19:45:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dafbf1b665e442ef8d91c814c00640f4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4 2024-11-13T19:45:39,765 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4, entries=150, sequenceid=196, filesize=30.4 K 2024-11-13T19:45:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/01e62a4358604b649cfba5e981dfe35a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a 2024-11-13T19:45:39,779 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T19:45:39,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/81d6bbf479714c8e9461e5588ffd8a47 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47 2024-11-13T19:45:39,787 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T19:45:39,788 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f989ae1f218e9e3abedc1d7da78f0f53 in 626ms, sequenceid=196, compaction requested=true 2024-11-13T19:45:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:39,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-13T19:45:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-13T19:45:39,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T19:45:39,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-11-13T19:45:39,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 789 msec 2024-11-13T19:45:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:39,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:39,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:39,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113273e8e5f2c5b41329ce6ecb0d43aaa54_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527199854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527199855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527199855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527199855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527199858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742503_1679 (size=14794) 2024-11-13T19:45:39,891 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:39,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113273e8e5f2c5b41329ce6ecb0d43aaa54_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113273e8e5f2c5b41329ce6ecb0d43aaa54_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:39,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a989085f0d524e6da902ca83c1e90ccb, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:39,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a989085f0d524e6da902ca83c1e90ccb is 175, key is test_row_0/A:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:39,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742504_1680 (size=39749) 2024-11-13T19:45:39,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a989085f0d524e6da902ca83c1e90ccb 2024-11-13T19:45:39,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/58bbfce3b50a4802bfc911f98a848d73 is 50, key is test_row_0/B:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:39,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742505_1681 (size=12151) 2024-11-13T19:45:39,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/58bbfce3b50a4802bfc911f98a848d73 2024-11-13T19:45:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527199975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527199975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527199975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527199977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527199977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/e0cafc1e93844abba1702ee0101fb3e3 is 50, key is test_row_0/C:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:40,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742506_1682 (size=12151) 2024-11-13T19:45:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:45:40,144 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:40,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T19:45:40,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:40,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:40,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527200195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527200195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527200196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527200197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527200201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:40,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/e0cafc1e93844abba1702ee0101fb3e3 2024-11-13T19:45:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:40,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a989085f0d524e6da902ca83c1e90ccb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb 2024-11-13T19:45:40,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb, entries=200, sequenceid=217, filesize=38.8 K 2024-11-13T19:45:40,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/58bbfce3b50a4802bfc911f98a848d73 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73 2024-11-13T19:45:40,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T19:45:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/e0cafc1e93844abba1702ee0101fb3e3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3 2024-11-13T19:45:40,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T19:45:40,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f989ae1f218e9e3abedc1d7da78f0f53 in 689ms, sequenceid=217, compaction requested=true 2024-11-13T19:45:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:40,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:45:40,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:45:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:40,521 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60743 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:45:40,521 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:40,521 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,522 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/65f92892d4ae4a4c8d2f89455561a4ce, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=59.3 K 2024-11-13T19:45:40,522 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164157 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:45:40,522 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f92892d4ae4a4c8d2f89455561a4ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:40,522 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:40,522 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,522 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0612bbb4b31344d79bfcbe1c94096fa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527136579 2024-11-13T19:45:40,522 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=160.3 K 2024-11-13T19:45:40,522 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e933d937735743ca885268bd354d936d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731527136662 2024-11-13T19:45:40,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb] 2024-11-13T19:45:40,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e62a4358604b649cfba5e981dfe35a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731527138844 2024-11-13T19:45:40,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 58bbfce3b50a4802bfc911f98a848d73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:40,524 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting de894cb7cd114fb9bb4bee277dac89f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:40,524 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69115c771d44738be8d0d4f2a328ff6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527136579 2024-11-13T19:45:40,525 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584835d77e804d3e8a73afea3eed4957, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731527136662 2024-11-13T19:45:40,525 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dafbf1b665e442ef8d91c814c00640f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731527138844 2024-11-13T19:45:40,526 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a989085f0d524e6da902ca83c1e90ccb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:40,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T19:45:40,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:40,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:40,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:40,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:40,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:40,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:40,551 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:40,586 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#589 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:40,587 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/9ed5eff52fa14d578fb53fc9a846b5b5 is 50, key is test_row_0/B:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:40,592 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113aac434832d9746f0aa63eb9247cc524f_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:40,595 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113aac434832d9746f0aa63eb9247cc524f_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:40,595 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aac434832d9746f0aa63eb9247cc524f_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:40,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b4f9d860ef4446fa61ca3dc6397faab_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527140526/Put/seqid=0 2024-11-13T19:45:40,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742508_1684 (size=4469) 2024-11-13T19:45:40,652 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#588 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:40,652 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/2b277a48e7ab4a67b7eae01189c0ad10 is 175, key is test_row_0/A:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:40,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742509_1685 (size=14794) 2024-11-13T19:45:40,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:40,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742507_1683 (size=12459) 2024-11-13T19:45:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527200598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b4f9d860ef4446fa61ca3dc6397faab_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b4f9d860ef4446fa61ca3dc6397faab_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527200660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527200661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527200667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527200668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/4a7a5348390045cba82a0e3999cd0424, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:40,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/4a7a5348390045cba82a0e3999cd0424 is 175, key is test_row_0/A:col10/1731527140526/Put/seqid=0 2024-11-13T19:45:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,702 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/9ed5eff52fa14d578fb53fc9a846b5b5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9ed5eff52fa14d578fb53fc9a846b5b5 2024-11-13T19:45:40,708 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 9ed5eff52fa14d578fb53fc9a846b5b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:40,708 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:40,708 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=11, startTime=1731527140519; duration=0sec 2024-11-13T19:45:40,709 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:40,709 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:40,709 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:45:40,725 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60743 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:45:40,725 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:40,726 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,726 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/574e562bcb5b461d8a9cb557cb70aa46, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=59.3 K 2024-11-13T19:45:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742510_1686 (size=31413) 2024-11-13T19:45:40,727 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 574e562bcb5b461d8a9cb557cb70aa46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731527136160 2024-11-13T19:45:40,727 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 34a3c9b00b7f4999a23f5ad9982ef444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731527136579 2024-11-13T19:45:40,727 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 91258d0149934acb8fce16f3ead67d05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731527136662 2024-11-13T19:45:40,728 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d6bbf479714c8e9461e5588ffd8a47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731527138844 2024-11-13T19:45:40,728 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e0cafc1e93844abba1702ee0101fb3e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742511_1687 (size=39749) 2024-11-13T19:45:40,750 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/4a7a5348390045cba82a0e3999cd0424 2024-11-13T19:45:40,756 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#591 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:40,757 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/4d5dba2eaa5d447ca3c0ed3a72b1439f is 50, key is test_row_0/C:col10/1731527139187/Put/seqid=0 2024-11-13T19:45:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:40,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b319642d99a141ae81df05578826685f is 50, key is test_row_0/B:col10/1731527140526/Put/seqid=0 2024-11-13T19:45:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742512_1688 (size=12459) 2024-11-13T19:45:40,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527200784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527200794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527200795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527200795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527200803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:40,810 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/4d5dba2eaa5d447ca3c0ed3a72b1439f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/4d5dba2eaa5d447ca3c0ed3a72b1439f 2024-11-13T19:45:40,815 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 4d5dba2eaa5d447ca3c0ed3a72b1439f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:40,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:40,815 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=11, startTime=1731527140520; duration=0sec 2024-11-13T19:45:40,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:40,815 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:40,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742513_1689 (size=12151) 2024-11-13T19:45:40,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:40,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:40,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527200993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527201004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527201015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527201015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527201015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,138 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/2b277a48e7ab4a67b7eae01189c0ad10 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10 2024-11-13T19:45:41,148 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into 2b277a48e7ab4a67b7eae01189c0ad10(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:41,148 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:41,148 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=11, startTime=1731527140519; duration=0sec 2024-11-13T19:45:41,148 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:41,148 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:41,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b319642d99a141ae81df05578826685f 2024-11-13T19:45:41,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a475978aabc14020b187dbc346739749 is 50, key is test_row_0/C:col10/1731527140526/Put/seqid=0 2024-11-13T19:45:41,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:41,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742514_1690 (size=12151) 2024-11-13T19:45:41,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a475978aabc14020b187dbc346739749 2024-11-13T19:45:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/4a7a5348390045cba82a0e3999cd0424 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424 2024-11-13T19:45:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527201304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424, entries=200, sequenceid=231, filesize=38.8 K 2024-11-13T19:45:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b319642d99a141ae81df05578826685f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f 2024-11-13T19:45:41,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f, entries=150, sequenceid=231, filesize=11.9 K 2024-11-13T19:45:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a475978aabc14020b187dbc346739749 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749 2024-11-13T19:45:41,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749, entries=150, sequenceid=231, filesize=11.9 K 2024-11-13T19:45:41,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f989ae1f218e9e3abedc1d7da78f0f53 in 803ms, sequenceid=231, compaction requested=false 2024-11-13T19:45:41,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:41,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T19:45:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:41,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527201378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f650d94ad16e4d3bb94ee6967d1fede9_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527201384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527201384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527201386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742515_1691 (size=14794) 2024-11-13T19:45:41,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:41,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f650d94ad16e4d3bb94ee6967d1fede9_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f650d94ad16e4d3bb94ee6967d1fede9_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:41,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/fe0042df63a748b7897fb5733b44142e, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:41,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/fe0042df63a748b7897fb5733b44142e is 175, key is test_row_0/A:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:41,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742516_1692 (size=39749) 2024-11-13T19:45:41,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527201494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/fe0042df63a748b7897fb5733b44142e 2024-11-13T19:45:41,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527201505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527201506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527201506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/67fff0635e2642799bef2aeb0c0a316d is 50, key is test_row_0/B:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:41,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742517_1693 (size=12151) 2024-11-13T19:45:41,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527201704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527201724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527201724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527201724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527201824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:41,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:41,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/67fff0635e2642799bef2aeb0c0a316d 2024-11-13T19:45:42,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b4d5e710b60b499c900c5ffb25bbaadc is 50, key is test_row_0/C:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527202023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527202034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527202043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527202043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742518_1694 (size=12151) 2024-11-13T19:45:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:42,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:42,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b4d5e710b60b499c900c5ffb25bbaadc 2024-11-13T19:45:42,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/fe0042df63a748b7897fb5733b44142e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e 2024-11-13T19:45:42,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e, entries=200, sequenceid=256, filesize=38.8 K 2024-11-13T19:45:42,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/67fff0635e2642799bef2aeb0c0a316d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d 2024-11-13T19:45:42,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T19:45:42,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b4d5e710b60b499c900c5ffb25bbaadc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc 2024-11-13T19:45:42,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T19:45:42,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for f989ae1f218e9e3abedc1d7da78f0f53 in 1182ms, sequenceid=256, compaction requested=true 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:42,523 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:42,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:42,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:42,526 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:42,526 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:42,526 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,526 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9ed5eff52fa14d578fb53fc9a846b5b5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=35.9 K 2024-11-13T19:45:42,526 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:42,526 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:42,526 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,527 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=108.3 K 2024-11-13T19:45:42,527 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,527 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e] 2024-11-13T19:45:42,527 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed5eff52fa14d578fb53fc9a846b5b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:42,527 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b277a48e7ab4a67b7eae01189c0ad10, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:42,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7a5348390045cba82a0e3999cd0424, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527139848 2024-11-13T19:45:42,528 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b319642d99a141ae81df05578826685f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527139848 2024-11-13T19:45:42,528 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0042df63a748b7897fb5733b44142e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527140659 2024-11-13T19:45:42,528 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fff0635e2642799bef2aeb0c0a316d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527141334 2024-11-13T19:45:42,549 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#597 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:42,550 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:42,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:45:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,556 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T19:45:42,557 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/e7ee1769af6a4605ba8d322d73e2e960 is 50, key is test_row_0/B:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:42,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113b05bc058638b403db1ab273b7120254e_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:42,563 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113b05bc058638b403db1ab273b7120254e_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:42,563 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b05bc058638b403db1ab273b7120254e_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742519_1695 (size=12561) 2024-11-13T19:45:42,611 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/e7ee1769af6a4605ba8d322d73e2e960 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e7ee1769af6a4605ba8d322d73e2e960 2024-11-13T19:45:42,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139085bf633d8f4057a87fc2e1c92feb5b_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527141381/Put/seqid=0 2024-11-13T19:45:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742520_1696 (size=4469) 2024-11-13T19:45:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742521_1697 (size=14994) 2024-11-13T19:45:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:42,654 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into e7ee1769af6a4605ba8d322d73e2e960(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:42,654 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:42,654 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=13, startTime=1731527142523; duration=0sec 2024-11-13T19:45:42,654 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:42,654 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:42,654 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:42,658 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139085bf633d8f4057a87fc2e1c92feb5b_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139085bf633d8f4057a87fc2e1c92feb5b_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:42,658 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:42,658 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:42,658 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:42,658 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/4d5dba2eaa5d447ca3c0ed3a72b1439f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=35.9 K 2024-11-13T19:45:42,659 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5dba2eaa5d447ca3c0ed3a72b1439f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731527139187 2024-11-13T19:45:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6b33f2f65d5b4a3594d80d6c28e0eb56, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:42,660 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a475978aabc14020b187dbc346739749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731527139848 2024-11-13T19:45:42,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6b33f2f65d5b4a3594d80d6c28e0eb56 is 175, key is test_row_0/A:col10/1731527141381/Put/seqid=0 2024-11-13T19:45:42,661 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d5e710b60b499c900c5ffb25bbaadc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527141334 2024-11-13T19:45:42,681 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#600 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:42,682 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/96a64e20952e401b8c98a0133a9a31f6 is 50, key is test_row_0/C:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742522_1698 (size=39949) 2024-11-13T19:45:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731527142679, max timestamp=1731527142708Num uniques 859; 2024-11-13T19:45:42,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T19:45:42,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742523_1699 (size=12561) 2024-11-13T19:45:42,732 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/96a64e20952e401b8c98a0133a9a31f6 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/96a64e20952e401b8c98a0133a9a31f6 2024-11-13T19:45:42,750 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 96a64e20952e401b8c98a0133a9a31f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:42,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:42,750 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=13, startTime=1731527142523; duration=0sec 2024-11-13T19:45:42,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:42,750 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527202733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527202739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527202739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527202755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527202845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,862 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527202864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:42,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527202875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,057 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#598 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:43,057 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dfd4e29821a24a859e84b61c7d1f6eb8 is 175, key is test_row_0/A:col10/1731527141339/Put/seqid=0 2024-11-13T19:45:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527203093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527203093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527203094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742524_1700 (size=31515) 2024-11-13T19:45:43,108 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6b33f2f65d5b4a3594d80d6c28e0eb56 2024-11-13T19:45:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3d60052f8466456a99ca391107ac2d18 is 50, key is test_row_0/B:col10/1731527141381/Put/seqid=0 2024-11-13T19:45:43,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742525_1701 (size=12301) 2024-11-13T19:45:43,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527203384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527203403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527203403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527203403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,507 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/dfd4e29821a24a859e84b61c7d1f6eb8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8 2024-11-13T19:45:43,513 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into dfd4e29821a24a859e84b61c7d1f6eb8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:43,513 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:43,513 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=13, startTime=1731527142523; duration=0sec 2024-11-13T19:45:43,513 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:43,513 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:43,567 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3d60052f8466456a99ca391107ac2d18 2024-11-13T19:45:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2e56afac64aa4f0e970175e0fa7ddcf8 is 50, key is test_row_0/C:col10/1731527141381/Put/seqid=0 2024-11-13T19:45:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742526_1702 (size=12301) 2024-11-13T19:45:43,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527203894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527203914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527203914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527203923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:44,001 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2e56afac64aa4f0e970175e0fa7ddcf8 2024-11-13T19:45:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6b33f2f65d5b4a3594d80d6c28e0eb56 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56 2024-11-13T19:45:44,026 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56, entries=200, sequenceid=268, filesize=39.0 K 2024-11-13T19:45:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3d60052f8466456a99ca391107ac2d18 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18 2024-11-13T19:45:44,035 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T19:45:44,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2e56afac64aa4f0e970175e0fa7ddcf8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8 2024-11-13T19:45:44,047 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T19:45:44,048 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=116.29 KB/119080 for f989ae1f218e9e3abedc1d7da78f0f53 in 1492ms, sequenceid=268, compaction requested=false 2024-11-13T19:45:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T19:45:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T19:45:44,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T19:45:44,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9010 sec 2024-11-13T19:45:44,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.9070 sec 2024-11-13T19:45:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:45:44,304 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:44,305 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T19:45:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:45:44,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:44,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:44,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:45:44,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T19:45:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:44,466 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=116.29 KB heapSize=341.80 KB 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d9a605a6a3f742f1b7fc34a40a7d87dc_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527142679/Put/seqid=0 2024-11-13T19:45:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742527_1703 (size=12454) 2024-11-13T19:45:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:44,552 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d9a605a6a3f742f1b7fc34a40a7d87dc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9a605a6a3f742f1b7fc34a40a7d87dc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/26684e49fae0491fbe4b777b8f02a6e4, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/26684e49fae0491fbe4b777b8f02a6e4 is 175, key is test_row_0/A:col10/1731527142679/Put/seqid=0 2024-11-13T19:45:44,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742528_1704 (size=31255) 2024-11-13T19:45:44,587 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/26684e49fae0491fbe4b777b8f02a6e4 2024-11-13T19:45:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/23500622016b425a8738736fc3e19038 is 50, key is test_row_0/B:col10/1731527142679/Put/seqid=0 2024-11-13T19:45:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:45:44,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742529_1705 (size=12301) 2024-11-13T19:45:44,639 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/23500622016b425a8738736fc3e19038 2024-11-13T19:45:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/85209c9d7d1147089084517e4bf68e99 is 50, key is test_row_0/C:col10/1731527142679/Put/seqid=0 2024-11-13T19:45:44,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742530_1706 (size=12301) 2024-11-13T19:45:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:44,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527204923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527204928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:45:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527204934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527204934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:44,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:44,940 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:44,940 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527204934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:44,946 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527205034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:45,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527205044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:45,098 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/85209c9d7d1147089084517e4bf68e99 2024-11-13T19:45:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/26684e49fae0491fbe4b777b8f02a6e4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4 2024-11-13T19:45:45,107 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4, entries=150, sequenceid=296, filesize=30.5 K 2024-11-13T19:45:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/23500622016b425a8738736fc3e19038 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038 2024-11-13T19:45:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,112 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T19:45:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/85209c9d7d1147089084517e4bf68e99 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99 2024-11-13T19:45:45,116 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99, entries=150, sequenceid=296, filesize=12.0 K 2024-11-13T19:45:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,117 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~341.75 KB/349952, currentSize=60.38 KB/61830 for f989ae1f218e9e3abedc1d7da78f0f53 in 651ms, sequenceid=296, compaction requested=true 2024-11-13T19:45:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T19:45:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T19:45:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T19:45:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-11-13T19:45:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 821 msec 2024-11-13T19:45:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:45,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T19:45:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:45,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f7b4db2ab59d44ec8ac9d0dd7778d624_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742531_1707 (size=12454) 2024-11-13T19:45:45,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f7b4db2ab59d44ec8ac9d0dd7778d624_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7b4db2ab59d44ec8ac9d0dd7778d624_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:45,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/744c9d92385e489b8ecaaf288f809e01, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/744c9d92385e489b8ecaaf288f809e01 is 175, key is test_row_0/A:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742532_1708 (size=31251) 2024-11-13T19:45:45,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/744c9d92385e489b8ecaaf288f809e01 2024-11-13T19:45:45,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f301157dd51e4b209922f586607e042a is 50, key is test_row_0/B:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527205350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:45,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527205352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:45,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742533_1709 (size=9857) 2024-11-13T19:45:45,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f301157dd51e4b209922f586607e042a 2024-11-13T19:45:45,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 is 50, key is test_row_0/C:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742534_1710 (size=9857) 2024-11-13T19:45:45,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 2024-11-13T19:45:45,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/744c9d92385e489b8ecaaf288f809e01 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01 2024-11-13T19:45:45,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01, entries=150, sequenceid=309, filesize=30.5 K 2024-11-13T19:45:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f301157dd51e4b209922f586607e042a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a 2024-11-13T19:45:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:45:45,444 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:45,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T19:45:45,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a, entries=100, sequenceid=309, filesize=9.6 K 2024-11-13T19:45:45,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 2024-11-13T19:45:45,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:45,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:45,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96, entries=100, sequenceid=309, filesize=9.6 K 2024-11-13T19:45:45,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f989ae1f218e9e3abedc1d7da78f0f53 in 195ms, sequenceid=309, compaction requested=true 2024-11-13T19:45:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:45,455 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:45,456 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:45,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:45,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:45,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:45,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:45,457 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:45,458 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:45,458 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,458 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=130.8 K 2024-11-13T19:45:45,458 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,458 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01] 2024-11-13T19:45:45,459 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:45,459 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:45,459 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,459 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e7ee1769af6a4605ba8d322d73e2e960, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=45.9 K 2024-11-13T19:45:45,459 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd4e29821a24a859e84b61c7d1f6eb8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527141334 2024-11-13T19:45:45,461 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ee1769af6a4605ba8d322d73e2e960, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527141334 2024-11-13T19:45:45,461 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b33f2f65d5b4a3594d80d6c28e0eb56, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731527141371 2024-11-13T19:45:45,461 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d60052f8466456a99ca391107ac2d18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731527141371 2024-11-13T19:45:45,462 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26684e49fae0491fbe4b777b8f02a6e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527142679 2024-11-13T19:45:45,462 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 23500622016b425a8738736fc3e19038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527142679 2024-11-13T19:45:45,462 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744c9d92385e489b8ecaaf288f809e01, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527144897 2024-11-13T19:45:45,462 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f301157dd51e4b209922f586607e042a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527144925 2024-11-13T19:45:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:45,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145334, max timestamp=1731527145467Num uniques 859; 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145334, max timestamp=1731527145467Num uniques 859; 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145334, max timestamp=1731527145467Num uniques 859; 2024-11-13T19:45:45,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:45,473 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,476 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#610 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:45,476 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3777c3901502483182389c19f9ac7b15 is 50, key is test_row_0/B:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,482 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113eac44def3dcf4373a9bccc302a32d0a1_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,484 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113eac44def3dcf4373a9bccc302a32d0a1_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,484 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113eac44def3dcf4373a9bccc302a32d0a1_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113390faee0327b4dafa7c5733928067643_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527145341/Put/seqid=0 2024-11-13T19:45:45,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742535_1711 (size=12847) 2024-11-13T19:45:45,510 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3777c3901502483182389c19f9ac7b15 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3777c3901502483182389c19f9ac7b15 2024-11-13T19:45:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742536_1712 (size=4469) 2024-11-13T19:45:45,520 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#609 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:45,520 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 3777c3901502483182389c19f9ac7b15(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:45,520 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:45,520 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=12, startTime=1731527145456; duration=0sec 2024-11-13T19:45:45,520 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:45,520 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:45,520 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:45,520 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/2bd1ee1df2ad4456b0a4465795e14259 is 175, key is test_row_0/A:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:45,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:45,523 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,523 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/96a64e20952e401b8c98a0133a9a31f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=45.9 K 2024-11-13T19:45:45,523 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a64e20952e401b8c98a0133a9a31f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731527141334 2024-11-13T19:45:45,524 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e56afac64aa4f0e970175e0fa7ddcf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731527141371 2024-11-13T19:45:45,525 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 85209c9d7d1147089084517e4bf68e99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731527142679 2024-11-13T19:45:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742537_1713 (size=12454) 2024-11-13T19:45:45,526 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b30a5e8bb5d42a7baa3e2a4bb69cf96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527144925 2024-11-13T19:45:45,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145507, max timestamp=1731527145547Num uniques 859; 2024-11-13T19:45:45,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145507, max timestamp=1731527145547Num uniques 859; 2024-11-13T19:45:45,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731527145507, max timestamp=1731527145547Num uniques 859; 2024-11-13T19:45:45,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T19:45:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:45,558 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#612 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:45,559 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/c248b967896f4388b4dc25c69626f9fe is 50, key is test_row_0/C:col10/1731527145259/Put/seqid=0 2024-11-13T19:45:45,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742538_1714 (size=31908) 2024-11-13T19:45:45,575 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/2bd1ee1df2ad4456b0a4465795e14259 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259 2024-11-13T19:45:45,579 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into 2bd1ee1df2ad4456b0a4465795e14259(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:45,579 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:45,580 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=12, startTime=1731527145455; duration=0sec 2024-11-13T19:45:45,580 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:45,580 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742539_1715 (size=12847) 2024-11-13T19:45:45,599 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/c248b967896f4388b4dc25c69626f9fe as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/c248b967896f4388b4dc25c69626f9fe 2024-11-13T19:45:45,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,604 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into c248b967896f4388b4dc25c69626f9fe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:45,605 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:45,605 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=12, startTime=1731527145456; duration=0sec 2024-11-13T19:45:45,605 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:45,605 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:45,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527145607, max timestamp=1731527145650Num uniques 1068; 2024-11-13T19:45:45,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T19:45:45,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527145607, max timestamp=1731527145650Num uniques 1068; 2024-11-13T19:45:45,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T19:45:45,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731527145607, max timestamp=1731527145650Num uniques 1068; 2024-11-13T19:45:45,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T19:45:45,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731527145718, max timestamp=1731527145742Num uniques 1027; 2024-11-13T19:45:45,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T19:45:45,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527145718, max timestamp=1731527145742Num uniques 1027; 2024-11-13T19:45:45,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T19:45:45,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731527145718, max timestamp=1731527145750Num uniques 1027; 2024-11-13T19:45:45,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T19:45:45,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527145776, max timestamp=1731527145816Num uniques 1036; 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527145776, max timestamp=1731527145816Num uniques 1036; 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731527145776, max timestamp=1731527145816Num uniques 1086; 2024-11-13T19:45:45,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T19:45:45,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527145864, max timestamp=1731527145884Num uniques 1045; 2024-11-13T19:45:45,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-13T19:45:45,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527145864, max timestamp=1731527145884Num uniques 1045; 2024-11-13T19:45:45,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-13T19:45:45,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731527145864, max timestamp=1731527145884Num uniques 1045; 2024-11-13T19:45:45,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-13T19:45:45,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:45,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:45,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:45,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113390faee0327b4dafa7c5733928067643_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113390faee0327b4dafa7c5733928067643_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:45,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/7e01d37ea91d4442af9a82fa06ed0cc7, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:45,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/7e01d37ea91d4442af9a82fa06ed0cc7 is 175, key is test_row_0/A:col10/1731527145341/Put/seqid=0 2024-11-13T19:45:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742540_1716 (size=31255) 2024-11-13T19:45:45,940 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/7e01d37ea91d4442af9a82fa06ed0cc7 2024-11-13T19:45:45,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/49f950b7682c46f5bd09444bbb76b788 is 50, key is test_row_0/B:col10/1731527145341/Put/seqid=0 2024-11-13T19:45:45,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742541_1717 (size=12301) 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527145946, max timestamp=1731527145966Num uniques 1054; 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527145946, max timestamp=1731527145966Num uniques 1054; 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731527145946, max timestamp=1731527145966Num uniques 1054; 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:45:45,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-13T19:45:46,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527146015, max timestamp=1731527146038Num uniques 1013; 2024-11-13T19:45:46,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-13T19:45:46,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527146015, max timestamp=1731527146038Num uniques 1013; 2024-11-13T19:45:46,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-13T19:45:46,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731527146015, max timestamp=1731527146038Num uniques 1013; 2024-11-13T19:45:46,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-13T19:45:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:46,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527146096, max timestamp=1731527146106Num uniques 1072; 2024-11-13T19:45:46,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-13T19:45:46,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527146096, max timestamp=1731527146106Num uniques 1072; 2024-11-13T19:45:46,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-13T19:45:46,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731527146096, max timestamp=1731527146106Num uniques 1072; 2024-11-13T19:45:46,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-13T19:45:46,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731527146135, max timestamp=1731527146171Num uniques 1031; 2024-11-13T19:45:46,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731527146135, max timestamp=1731527146171Num uniques 1031; 2024-11-13T19:45:46,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-13T19:45:46,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-13T19:45:46,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731527146135, max timestamp=1731527146171Num uniques 1031; 2024-11-13T19:45:46,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-13T19:45:46,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731527146205, max timestamp=1731527146241Num uniques 1090; 2024-11-13T19:45:46,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731527146205, max timestamp=1731527146241Num uniques 1090; 2024-11-13T19:45:46,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-13T19:45:46,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-13T19:45:46,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731527146230, max timestamp=1731527146241Num uniques 1090; 2024-11-13T19:45:46,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-13T19:45:46,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527146303, max timestamp=1731527146322Num uniques 1099; 2024-11-13T19:45:46,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-13T19:45:46,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527146303, max timestamp=1731527146322Num uniques 1099; 2024-11-13T19:45:46,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-13T19:45:46,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731527146303, max timestamp=1731527146322Num uniques 1049; 2024-11-13T19:45:46,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-13T19:45:46,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/49f950b7682c46f5bd09444bbb76b788 2024-11-13T19:45:46,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0732d0ca834147a887c4bcb013b19935 is 50, key is test_row_0/C:col10/1731527145341/Put/seqid=0 2024-11-13T19:45:46,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742542_1718 (size=12301) 2024-11-13T19:45:46,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0732d0ca834147a887c4bcb013b19935 2024-11-13T19:45:46,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527146364, max timestamp=1731527146411Num uniques 1058; 2024-11-13T19:45:46,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-13T19:45:46,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527146364, max timestamp=1731527146411Num uniques 1058; 2024-11-13T19:45:46,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-13T19:45:46,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731527146364, max timestamp=1731527146411Num uniques 1058; 2024-11-13T19:45:46,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/54d0784d1943:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-13T19:45:46,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/7e01d37ea91d4442af9a82fa06ed0cc7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7 2024-11-13T19:45:46,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7, entries=150, sequenceid=330, filesize=30.5 K 2024-11-13T19:45:46,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/49f950b7682c46f5bd09444bbb76b788 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788 2024-11-13T19:45:46,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T19:45:46,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0732d0ca834147a887c4bcb013b19935 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935 2024-11-13T19:45:46,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935, entries=150, sequenceid=330, filesize=12.0 K 2024-11-13T19:45:46,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=93.93 KB/96180 for f989ae1f218e9e3abedc1d7da78f0f53 in 1000ms, sequenceid=330, compaction requested=false 2024-11-13T19:45:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=100.63 KB heapSize=285.47 KB 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T19:45:46,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb128477927a4a3a8be6f0588340f39e_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742543_1719 (size=17534) 2024-11-13T19:45:46,487 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:46,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb128477927a4a3a8be6f0588340f39e_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb128477927a4a3a8be6f0588340f39e_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ce0061884f5b4d959987cae60d73585b, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ce0061884f5b4d959987cae60d73585b is 175, key is test_row_0/A:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742544_1720 (size=48639) 2024-11-13T19:45:46,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=554, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ce0061884f5b4d959987cae60d73585b 2024-11-13T19:45:46,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8724cb99e95a408eb5acd3a6d55f6616 is 50, key is test_row_0/B:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742545_1721 (size=12301) 2024-11-13T19:45:46,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8724cb99e95a408eb5acd3a6d55f6616 2024-11-13T19:45:46,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/35150b0f514449d6a08544ffd999215e is 50, key is test_row_0/C:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742546_1722 (size=12301) 2024-11-13T19:45:46,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/35150b0f514449d6a08544ffd999215e 2024-11-13T19:45:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ce0061884f5b4d959987cae60d73585b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b 2024-11-13T19:45:46,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b, entries=250, sequenceid=554, filesize=47.5 K 2024-11-13T19:45:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527206539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8724cb99e95a408eb5acd3a6d55f6616 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616 2024-11-13T19:45:46,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527206541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616, entries=150, sequenceid=554, filesize=12.0 K 2024-11-13T19:45:46,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/35150b0f514449d6a08544ffd999215e as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e 2024-11-13T19:45:46,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e, entries=150, sequenceid=554, filesize=12.0 K 2024-11-13T19:45:46,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~285.42 KB/292272, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 91ms, sequenceid=554, compaction requested=true 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:46,559 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:46,559 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:46,560 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111802 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:46,560 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:46,560 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:46,560 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:46,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,560 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=109.2 K 2024-11-13T19:45:46,560 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3777c3901502483182389c19f9ac7b15, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=36.6 K 2024-11-13T19:45:46,560 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,560 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b] 2024-11-13T19:45:46,561 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3777c3901502483182389c19f9ac7b15, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527142729 2024-11-13T19:45:46,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd1ee1df2ad4456b0a4465795e14259, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527142729 2024-11-13T19:45:46,561 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e01d37ea91d4442af9a82fa06ed0cc7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731527145341 2024-11-13T19:45:46,561 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f950b7682c46f5bd09444bbb76b788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731527145341 2024-11-13T19:45:46,562 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8724cb99e95a408eb5acd3a6d55f6616, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731527146445 2024-11-13T19:45:46,562 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce0061884f5b4d959987cae60d73585b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731527146445 2024-11-13T19:45:46,575 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#618 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:46,576 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/947d83f612db46c796e3db9e1d5b8849 is 50, key is test_row_0/B:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,578 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:46,599 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113376d694fba7140a9b680276e9b4a8343_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,601 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113376d694fba7140a9b680276e9b4a8343_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,602 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113376d694fba7140a9b680276e9b4a8343_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742547_1723 (size=12949) 2024-11-13T19:45:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742548_1724 (size=4469) 2024-11-13T19:45:46,618 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#619 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:46,618 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d1e6ab3f1f1d4a029faca001a3f0f848 is 175, key is test_row_0/A:col10/1731527146465/Put/seqid=0 2024-11-13T19:45:46,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742549_1725 (size=31903) 2024-11-13T19:45:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:46,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f5bc815453ab49b6ac7ffe50f9efb742_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527146656/Put/seqid=0 2024-11-13T19:45:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742550_1726 (size=14994) 2024-11-13T19:45:46,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:46,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f5bc815453ab49b6ac7ffe50f9efb742_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f5bc815453ab49b6ac7ffe50f9efb742_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d0f369ceb80e4e9b891c47a1f1e4bef4, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527206709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d0f369ceb80e4e9b891c47a1f1e4bef4 is 175, key is test_row_0/A:col10/1731527146656/Put/seqid=0 2024-11-13T19:45:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527206715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742551_1727 (size=39949) 2024-11-13T19:45:46,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=571, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d0f369ceb80e4e9b891c47a1f1e4bef4 2024-11-13T19:45:46,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f869369a7159457c8073e7092c6a8c8a is 50, key is test_row_0/B:col10/1731527146656/Put/seqid=0 2024-11-13T19:45:46,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742552_1728 (size=12301) 2024-11-13T19:45:46,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f869369a7159457c8073e7092c6a8c8a 2024-11-13T19:45:46,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b7a6a774cf5a43b5aab222c6aeb88aaf is 50, key is test_row_0/C:col10/1731527146656/Put/seqid=0 2024-11-13T19:45:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527206824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527206837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:46,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742553_1729 (size=12301) 2024-11-13T19:45:46,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b7a6a774cf5a43b5aab222c6aeb88aaf 2024-11-13T19:45:46,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d0f369ceb80e4e9b891c47a1f1e4bef4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4 2024-11-13T19:45:46,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:46,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:46,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4, entries=200, sequenceid=571, filesize=39.0 K 2024-11-13T19:45:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f869369a7159457c8073e7092c6a8c8a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a 2024-11-13T19:45:46,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a, entries=150, sequenceid=571, filesize=12.0 K 2024-11-13T19:45:46,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b7a6a774cf5a43b5aab222c6aeb88aaf as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf 2024-11-13T19:45:46,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf, entries=150, sequenceid=571, filesize=12.0 K 2024-11-13T19:45:46,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 234ms, sequenceid=571, compaction requested=true 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:46,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:45:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:45:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:46,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113690a173f268b4f2981f88adfcc5d9455_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742554_1730 (size=12454) 2024-11-13T19:45:46,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:46,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113690a173f268b4f2981f88adfcc5d9455_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113690a173f268b4f2981f88adfcc5d9455_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:46,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ba5fb5ff0ff142eda9f2e3c727f966fc, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:46,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ba5fb5ff0ff142eda9f2e3c727f966fc is 175, key is test_row_0/A:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:46,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742555_1731 (size=31255) 2024-11-13T19:45:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527206995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,015 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/947d83f612db46c796e3db9e1d5b8849 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/947d83f612db46c796e3db9e1d5b8849 2024-11-13T19:45:47,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,023 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 947d83f612db46c796e3db9e1d5b8849(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,023 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=13, startTime=1731527146559; duration=0sec 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:47,023 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. because compaction request was cancelled 2024-11-13T19:45:47,024 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:47,024 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,025 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:47,025 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:47,025 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,025 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/c248b967896f4388b4dc25c69626f9fe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=48.6 K 2024-11-13T19:45:47,025 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c248b967896f4388b4dc25c69626f9fe, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731527142729 2024-11-13T19:45:47,028 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0732d0ca834147a887c4bcb013b19935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731527145341 2024-11-13T19:45:47,028 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 35150b0f514449d6a08544ffd999215e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731527146445 2024-11-13T19:45:47,028 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a6a774cf5a43b5aab222c6aeb88aaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731527146539 2024-11-13T19:45:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527207034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,041 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d1e6ab3f1f1d4a029faca001a3f0f848 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848 2024-11-13T19:45:47,043 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#624 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:47,043 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/08af832b8aab47cab71e07dee027131f is 50, key is test_row_0/C:col10/1731527146656/Put/seqid=0 2024-11-13T19:45:47,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527207055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,062 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into d1e6ab3f1f1d4a029faca001a3f0f848(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:47,062 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,062 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=13, startTime=1731527146559; duration=0sec 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. because compaction request was cancelled 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:47,063 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:45:47,066 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:47,066 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:47,066 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. because compaction request was cancelled 2024-11-13T19:45:47,066 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:47,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742556_1732 (size=12983) 2024-11-13T19:45:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527207114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527207323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527207346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527207363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,402 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=588, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ba5fb5ff0ff142eda9f2e3c727f966fc 2024-11-13T19:45:47,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/4c46e2fd97514567b13a6ee06f1961ba is 50, key is test_row_0/B:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:47,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742557_1733 (size=12301) 2024-11-13T19:45:47,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/4c46e2fd97514567b13a6ee06f1961ba 2024-11-13T19:45:47,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a911420c6ca24f16b84233e482a4babe is 50, key is test_row_0/C:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:47,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742558_1734 (size=12301) 2024-11-13T19:45:47,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a911420c6ca24f16b84233e482a4babe 2024-11-13T19:45:47,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ba5fb5ff0ff142eda9f2e3c727f966fc as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc 2024-11-13T19:45:47,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc, entries=150, sequenceid=588, filesize=30.5 K 2024-11-13T19:45:47,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/4c46e2fd97514567b13a6ee06f1961ba as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba 2024-11-13T19:45:47,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba, entries=150, sequenceid=588, filesize=12.0 K 2024-11-13T19:45:47,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/a911420c6ca24f16b84233e482a4babe as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe 2024-11-13T19:45:47,506 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/08af832b8aab47cab71e07dee027131f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/08af832b8aab47cab71e07dee027131f 2024-11-13T19:45:47,512 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 08af832b8aab47cab71e07dee027131f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:47,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,512 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=12, startTime=1731527146892; duration=0sec 2024-11-13T19:45:47,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:47,512 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:47,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe, entries=150, sequenceid=588, filesize=12.0 K 2024-11-13T19:45:47,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 567ms, sequenceid=588, compaction requested=true 2024-11-13T19:45:47,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,515 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:47,516 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:47,517 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:47,517 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:47,517 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,517 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:47,518 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/947d83f612db46c796e3db9e1d5b8849, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=36.7 K 2024-11-13T19:45:47,518 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:47,518 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,518 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=100.7 K 2024-11-13T19:45:47,518 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,518 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc] 2024-11-13T19:45:47,518 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 947d83f612db46c796e3db9e1d5b8849, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731527146445 2024-11-13T19:45:47,518 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting f869369a7159457c8073e7092c6a8c8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731527146539 2024-11-13T19:45:47,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e6ab3f1f1d4a029faca001a3f0f848, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731527146445 2024-11-13T19:45:47,519 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c46e2fd97514567b13a6ee06f1961ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731527146711 2024-11-13T19:45:47,519 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0f369ceb80e4e9b891c47a1f1e4bef4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731527146537 2024-11-13T19:45:47,527 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5fb5ff0ff142eda9f2e3c727f966fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731527146711 2024-11-13T19:45:47,528 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#627 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:47,529 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/95cb9fdab1ad45a2b463c3bf5ddfd47b is 50, key is test_row_0/B:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:47,538 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742559_1735 (size=13051) 2024-11-13T19:45:47,564 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/95cb9fdab1ad45a2b463c3bf5ddfd47b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/95cb9fdab1ad45a2b463c3bf5ddfd47b 2024-11-13T19:45:47,567 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136486b7b8205a480185a3d3ae73011967_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,569 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136486b7b8205a480185a3d3ae73011967_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,569 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136486b7b8205a480185a3d3ae73011967_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,571 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 95cb9fdab1ad45a2b463c3bf5ddfd47b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:47,571 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,571 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=13, startTime=1731527147516; duration=0sec 2024-11-13T19:45:47,571 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:47,571 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:47,571 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:45:47,572 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T19:45:47,572 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T19:45:47,572 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. because compaction request was cancelled 2024-11-13T19:45:47,572 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:47,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742560_1736 (size=4469) 2024-11-13T19:45:47,576 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#628 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:47,577 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ef6e966b27b14d23885e340b1df55193 is 175, key is test_row_0/A:col10/1731527146947/Put/seqid=0 2024-11-13T19:45:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742561_1737 (size=32005) 2024-11-13T19:45:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:47,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:45:47,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,640 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T19:45:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:47,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d94523416f0f4b769e040be12edabba8_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527146999/Put/seqid=0 2024-11-13T19:45:47,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742562_1738 (size=14994) 2024-11-13T19:45:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:47,666 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d94523416f0f4b769e040be12edabba8_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d94523416f0f4b769e040be12edabba8_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:47,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/0691ce4fc1074c2d821d8b6197eb7ab7, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/0691ce4fc1074c2d821d8b6197eb7ab7 is 175, key is test_row_0/A:col10/1731527146999/Put/seqid=0 2024-11-13T19:45:47,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742563_1739 (size=39949) 2024-11-13T19:45:47,673 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=608, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/0691ce4fc1074c2d821d8b6197eb7ab7 2024-11-13T19:45:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbfdbdc3b6264448a224008c6179d474 is 50, key is test_row_0/B:col10/1731527146999/Put/seqid=0 2024-11-13T19:45:47,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742564_1740 (size=12301) 2024-11-13T19:45:47,684 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbfdbdc3b6264448a224008c6179d474 2024-11-13T19:45:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527207679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 is 50, key is test_row_0/C:col10/1731527146999/Put/seqid=0 2024-11-13T19:45:47,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742565_1741 (size=12301) 2024-11-13T19:45:47,698 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 2024-11-13T19:45:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/0691ce4fc1074c2d821d8b6197eb7ab7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7 2024-11-13T19:45:47,706 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7, entries=200, sequenceid=608, filesize=39.0 K 2024-11-13T19:45:47,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbfdbdc3b6264448a224008c6179d474 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474 2024-11-13T19:45:47,711 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474, entries=150, sequenceid=608, filesize=12.0 K 2024-11-13T19:45:47,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 2024-11-13T19:45:47,719 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8, entries=150, sequenceid=608, filesize=12.0 K 2024-11-13T19:45:47,721 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f989ae1f218e9e3abedc1d7da78f0f53 in 81ms, sequenceid=608, compaction requested=true 2024-11-13T19:45:47,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:47,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:47,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T19:45:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T19:45:47,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T19:45:47,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-11-13T19:45:47,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2810 sec 2024-11-13T19:45:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:47,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:47,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:47,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362aeb0785b144c38bc3cf3c191325feb_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:47,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742566_1742 (size=12454) 2024-11-13T19:45:47,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:47,835 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111362aeb0785b144c38bc3cf3c191325feb_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362aeb0785b144c38bc3cf3c191325feb_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:47,836 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/aec080e0ad004a81b95386547ae312fa, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:47,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/aec080e0ad004a81b95386547ae312fa is 175, key is test_row_0/A:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527207842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527207865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742567_1743 (size=31255) 2024-11-13T19:45:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527207874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527207954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527207954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527207974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:47,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:47,996 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/ef6e966b27b14d23885e340b1df55193 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193 2024-11-13T19:45:48,002 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into ef6e966b27b14d23885e340b1df55193(size=31.3 K), total size for store is 70.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:48,002 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:48,003 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=13, startTime=1731527147515; duration=0sec 2024-11-13T19:45:48,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:48,003 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527208164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527208164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527208194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,285 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=625, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/aec080e0ad004a81b95386547ae312fa 2024-11-13T19:45:48,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8cf8736e24d040a791338f63cf109ced is 50, key is test_row_0/B:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742568_1744 (size=12301) 2024-11-13T19:45:48,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8cf8736e24d040a791338f63cf109ced 2024-11-13T19:45:48,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0b36d8ff30594e3186ce3b81d589000d is 50, key is test_row_0/C:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742569_1745 (size=12301) 2024-11-13T19:45:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527208475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527208478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527208503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0b36d8ff30594e3186ce3b81d589000d 2024-11-13T19:45:48,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/aec080e0ad004a81b95386547ae312fa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa 2024-11-13T19:45:48,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa, entries=150, sequenceid=625, filesize=30.5 K 2024-11-13T19:45:48,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/8cf8736e24d040a791338f63cf109ced as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced 2024-11-13T19:45:48,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced, entries=150, sequenceid=625, filesize=12.0 K 2024-11-13T19:45:48,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/0b36d8ff30594e3186ce3b81d589000d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d 2024-11-13T19:45:48,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d, entries=150, sequenceid=625, filesize=12.0 K 2024-11-13T19:45:48,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 1071ms, sequenceid=625, compaction requested=true 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:48,868 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:48,868 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:48,875 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:48,875 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:48,875 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:48,875 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/95cb9fdab1ad45a2b463c3bf5ddfd47b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=36.8 K 2024-11-13T19:45:48,877 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:48,877 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cb9fdab1ad45a2b463c3bf5ddfd47b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731527146711 2024-11-13T19:45:48,877 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:48,877 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:48,877 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=100.8 K 2024-11-13T19:45:48,877 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:48,877 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa] 2024-11-13T19:45:48,885 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bbfdbdc3b6264448a224008c6179d474, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731527146998 2024-11-13T19:45:48,886 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6e966b27b14d23885e340b1df55193, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731527146711 2024-11-13T19:45:48,886 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf8736e24d040a791338f63cf109ced, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:48,887 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0691ce4fc1074c2d821d8b6197eb7ab7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731527146986 2024-11-13T19:45:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:48,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T19:45:48,888 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec080e0ad004a81b95386547ae312fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:48,900 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:48,901 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#635 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:48,901 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/9c375a8a4b40437780588ca4835503f3 is 50, key is test_row_0/B:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:48,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c3a650e7ff8b4c5bb673024daef3ceef_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527147841/Put/seqid=0 2024-11-13T19:45:48,905 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c1137bb20c294eaeb08ce8083d5a9e48_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:48,907 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c1137bb20c294eaeb08ce8083d5a9e48_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:48,908 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c1137bb20c294eaeb08ce8083d5a9e48_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:48,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742570_1746 (size=12454) 2024-11-13T19:45:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527208949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527208949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:48,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742571_1747 (size=13153) 2024-11-13T19:45:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742572_1748 (size=4469) 2024-11-13T19:45:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527208998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527209001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527209014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527209063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527209063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527209279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527209279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:49,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c3a650e7ff8b4c5bb673024daef3ceef_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3a650e7ff8b4c5bb673024daef3ceef_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:49,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/72423baf3e0b49b0a1ec79cb97d8b110, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:49,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/72423baf3e0b49b0a1ec79cb97d8b110 is 175, key is test_row_0/A:col10/1731527147841/Put/seqid=0 2024-11-13T19:45:49,390 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#636 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:49,391 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d608478ba2fd4410b838e62faacc672a is 175, key is test_row_0/A:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:49,396 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/9c375a8a4b40437780588ca4835503f3 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9c375a8a4b40437780588ca4835503f3 2024-11-13T19:45:49,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742573_1749 (size=31255) 2024-11-13T19:45:49,406 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 9c375a8a4b40437780588ca4835503f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:49,406 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:49,406 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=13, startTime=1731527148868; duration=0sec 2024-11-13T19:45:49,406 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:49,406 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:49,406 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:49,411 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:49,411 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:49,411 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:49,411 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/08af832b8aab47cab71e07dee027131f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=48.7 K 2024-11-13T19:45:49,412 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 08af832b8aab47cab71e07dee027131f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731527146539 2024-11-13T19:45:49,412 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting a911420c6ca24f16b84233e482a4babe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1731527146711 2024-11-13T19:45:49,412 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae3bb6c45b54ecc9d26e45b783ae9e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731527146998 2024-11-13T19:45:49,413 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b36d8ff30594e3186ce3b81d589000d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:49,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742574_1750 (size=32107) 2024-11-13T19:45:49,427 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/d608478ba2fd4410b838e62faacc672a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a 2024-11-13T19:45:49,428 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#638 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:49,429 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/58f0144c1ef042338d1fffdb6f01d872 is 50, key is test_row_0/C:col10/1731527147674/Put/seqid=0 2024-11-13T19:45:49,442 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into d608478ba2fd4410b838e62faacc672a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:49,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:49,442 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=13, startTime=1731527148868; duration=0sec 2024-11-13T19:45:49,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:49,442 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:49,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742575_1751 (size=13119) 2024-11-13T19:45:49,465 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/58f0144c1ef042338d1fffdb6f01d872 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/58f0144c1ef042338d1fffdb6f01d872 2024-11-13T19:45:49,471 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 58f0144c1ef042338d1fffdb6f01d872(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:49,471 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:49,471 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=12, startTime=1731527148868; duration=0sec 2024-11-13T19:45:49,471 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:49,471 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527209595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527209597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:45:49,604 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:49,605 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:45:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T19:45:49,608 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:45:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:49,609 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:45:49,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:45:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:49,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:49,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:49,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:49,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:49,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=642, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/72423baf3e0b49b0a1ec79cb97d8b110 2024-11-13T19:45:49,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/2af0230a4df54b04a20960cfaa0c086c is 50, key is test_row_0/B:col10/1731527147841/Put/seqid=0 2024-11-13T19:45:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742576_1752 (size=12301) 2024-11-13T19:45:49,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/2af0230a4df54b04a20960cfaa0c086c 2024-11-13T19:45:49,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/92190d22b404404faef1e237d0f5f5e5 is 50, key is test_row_0/C:col10/1731527147841/Put/seqid=0 2024-11-13T19:45:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:49,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:49,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742577_1753 (size=12301) 2024-11-13T19:45:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527210015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:50,023 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:50,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527210025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527210038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:50,044 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:50,045 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:50,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527210113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527210118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:50,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/92190d22b404404faef1e237d0f5f5e5 2024-11-13T19:45:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/72423baf3e0b49b0a1ec79cb97d8b110 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110 2024-11-13T19:45:50,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110, entries=150, sequenceid=642, filesize=30.5 K 2024-11-13T19:45:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/2af0230a4df54b04a20960cfaa0c086c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c 2024-11-13T19:45:50,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c, entries=150, sequenceid=642, filesize=12.0 K 2024-11-13T19:45:50,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/92190d22b404404faef1e237d0f5f5e5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5 2024-11-13T19:45:50,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5, entries=150, sequenceid=642, filesize=12.0 K 2024-11-13T19:45:50,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:45:50,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f989ae1f218e9e3abedc1d7da78f0f53 in 1541ms, sequenceid=642, compaction requested=false 2024-11-13T19:45:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:50,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36227 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:45:50,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:50,585 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:50,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b55c579d45714bdb82c5bd56c3d0afdd_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527148945/Put/seqid=0 2024-11-13T19:45:50,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742578_1754 (size=12454) 2024-11-13T19:45:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:51,057 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b55c579d45714bdb82c5bd56c3d0afdd_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b55c579d45714bdb82c5bd56c3d0afdd_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/9c20214141da46859dd0f570a915744b, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/9c20214141da46859dd0f570a915744b is 175, key is test_row_0/A:col10/1731527148945/Put/seqid=0 2024-11-13T19:45:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742579_1755 (size=31255) 2024-11-13T19:45:51,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. as already flushing 2024-11-13T19:45:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527211188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527211188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527211306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527211306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,463 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=662, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/9c20214141da46859dd0f570a915744b 2024-11-13T19:45:51,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527211525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527211533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/c94400b2e3cc4c27a7ae0df163ae7c83 is 50, key is test_row_0/B:col10/1731527148945/Put/seqid=0 2024-11-13T19:45:51,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742580_1756 (size=12301) 2024-11-13T19:45:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:51,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527211846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527211847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:51,990 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/c94400b2e3cc4c27a7ae0df163ae7c83 2024-11-13T19:45:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b9072b4ef16b4cb78c54c785444a0872 is 50, key is test_row_0/C:col10/1731527148945/Put/seqid=0 2024-11-13T19:45:52,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742581_1757 (size=12301) 2024-11-13T19:45:52,053 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b9072b4ef16b4cb78c54c785444a0872 2024-11-13T19:45:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37764 deadline: 1731527212045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,056 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_2' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/9c20214141da46859dd0f570a915744b as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b 2024-11-13T19:45:52,062 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b, entries=150, sequenceid=662, filesize=30.5 K 2024-11-13T19:45:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/c94400b2e3cc4c27a7ae0df163ae7c83 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83 2024-11-13T19:45:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83, entries=150, sequenceid=662, filesize=12.0 K 2024-11-13T19:45:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b9072b4ef16b4cb78c54c785444a0872 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872 2024-11-13T19:45:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37694 deadline: 1731527212066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,083 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872, entries=150, sequenceid=662, filesize=12.0 K 2024-11-13T19:45:52,082 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 54d0784d1943,36227,1731526989835 for 'test_row_0' in f989ae1f218e9e3abedc1d7da78f0f53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T19:45:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,084 INFO [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 1499ms, sequenceid=662, compaction requested=true 2024-11-13T19:45:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T19:45:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T19:45:52,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T19:45:52,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-13T19:45:52,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.4840 sec 2024-11-13T19:45:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:52,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:52,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cd57720eec504b27a43e0e80b9330e1f_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742582_1758 (size=14994) 2024-11-13T19:45:52,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:52,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cd57720eec504b27a43e0e80b9330e1f_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cd57720eec504b27a43e0e80b9330e1f_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:52,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a5a4f51b41914e51bd5cb4f47e1c162a, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a5a4f51b41914e51bd5cb4f47e1c162a is 175, key is test_row_0/A:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742583_1759 (size=39949) 2024-11-13T19:45:52,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=678, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a5a4f51b41914e51bd5cb4f47e1c162a 2024-11-13T19:45:52,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b952f03bc844456fb51428d91765750f is 50, key is test_row_0/B:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742584_1760 (size=12301) 2024-11-13T19:45:52,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b952f03bc844456fb51428d91765750f 2024-11-13T19:45:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527212323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/d0d75927a51d4a5aae1e41165f0f2ae4 is 50, key is test_row_0/C:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527212364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527212364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742585_1761 (size=12301) 2024-11-13T19:45:52,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/d0d75927a51d4a5aae1e41165f0f2ae4 2024-11-13T19:45:52,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/a5a4f51b41914e51bd5cb4f47e1c162a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a 2024-11-13T19:45:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a, entries=200, sequenceid=678, filesize=39.0 K 2024-11-13T19:45:52,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/b952f03bc844456fb51428d91765750f as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f 2024-11-13T19:45:52,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f, entries=150, sequenceid=678, filesize=12.0 K 2024-11-13T19:45:52,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/d0d75927a51d4a5aae1e41165f0f2ae4 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4 2024-11-13T19:45:52,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4, entries=150, sequenceid=678, filesize=12.0 K 2024-11-13T19:45:52,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f989ae1f218e9e3abedc1d7da78f0f53 in 319ms, sequenceid=678, compaction requested=true 2024-11-13T19:45:52,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:52,421 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:52,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:52,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:52,422 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:52,423 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:52,423 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:52,423 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:52,423 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=131.4 K 2024-11-13T19:45:52,423 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:52,423 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a] 2024-11-13T19:45:52,425 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:52,425 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:52,425 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:52,425 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9c375a8a4b40437780588ca4835503f3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=48.9 K 2024-11-13T19:45:52,425 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting d608478ba2fd4410b838e62faacc672a, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:52,425 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c375a8a4b40437780588ca4835503f3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:52,426 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72423baf3e0b49b0a1ec79cb97d8b110, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731527147832 2024-11-13T19:45:52,426 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af0230a4df54b04a20960cfaa0c086c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731527147832 2024-11-13T19:45:52,426 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c20214141da46859dd0f570a915744b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731527148938 2024-11-13T19:45:52,426 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting c94400b2e3cc4c27a7ae0df163ae7c83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731527148938 2024-11-13T19:45:52,426 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5a4f51b41914e51bd5cb4f47e1c162a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:52,427 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b952f03bc844456fb51428d91765750f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:52,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:52,454 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:52,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:52,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:52,467 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#648 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:52,467 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3225baf3b76c4da2a5cf175c8b7c695c is 50, key is test_row_0/B:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,485 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113a85442f10e8144d9beeb780c30f9db54_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,488 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113a85442f10e8144d9beeb780c30f9db54_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,488 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a85442f10e8144d9beeb780c30f9db54_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742586_1762 (size=13289) 2024-11-13T19:45:52,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131507771218264777bab62dbedd6e0735_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527152455/Put/seqid=0 2024-11-13T19:45:52,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742587_1763 (size=4469) 2024-11-13T19:45:52,586 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#647 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:52,586 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6c7f99b818dd460c9c77174f0b222751 is 175, key is test_row_0/A:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742588_1764 (size=12454) 2024-11-13T19:45:52,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:52,629 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/3225baf3b76c4da2a5cf175c8b7c695c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3225baf3b76c4da2a5cf175c8b7c695c 2024-11-13T19:45:52,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131507771218264777bab62dbedd6e0735_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131507771218264777bab62dbedd6e0735_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742589_1765 (size=32243) 2024-11-13T19:45:52,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/db7bc53ab656479f98cafab6adf48799, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:52,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/db7bc53ab656479f98cafab6adf48799 is 175, key is test_row_0/A:col10/1731527152455/Put/seqid=0 2024-11-13T19:45:52,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527212652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,665 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/6c7f99b818dd460c9c77174f0b222751 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751 2024-11-13T19:45:52,668 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into 3225baf3b76c4da2a5cf175c8b7c695c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:52,668 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:52,668 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=12, startTime=1731527152422; duration=0sec 2024-11-13T19:45:52,668 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:52,668 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:52,668 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:45:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742590_1766 (size=31255) 2024-11-13T19:45:52,690 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:45:52,690 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:52,690 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:52,690 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/58f0144c1ef042338d1fffdb6f01d872, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=48.8 K 2024-11-13T19:45:52,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=695, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/db7bc53ab656479f98cafab6adf48799 2024-11-13T19:45:52,691 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f0144c1ef042338d1fffdb6f01d872, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731527147664 2024-11-13T19:45:52,692 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 92190d22b404404faef1e237d0f5f5e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731527147832 2024-11-13T19:45:52,692 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b9072b4ef16b4cb78c54c785444a0872, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731527148938 2024-11-13T19:45:52,692 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d75927a51d4a5aae1e41165f0f2ae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:52,703 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into 6c7f99b818dd460c9c77174f0b222751(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:52,703 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:52,703 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=12, startTime=1731527152421; duration=0sec 2024-11-13T19:45:52,703 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:52,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbb3c71819ac4c9484ea8c7137bcaf1a is 50, key is test_row_0/B:col10/1731527152455/Put/seqid=0 2024-11-13T19:45:52,703 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:52,716 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#651 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:52,716 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/e43e77b8125046728809c6d54afbef54 is 50, key is test_row_0/C:col10/1731527152087/Put/seqid=0 2024-11-13T19:45:52,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742591_1767 (size=12301) 2024-11-13T19:45:52,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742592_1768 (size=13255) 2024-11-13T19:45:52,757 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/e43e77b8125046728809c6d54afbef54 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e43e77b8125046728809c6d54afbef54 2024-11-13T19:45:52,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527212784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:52,800 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into e43e77b8125046728809c6d54afbef54(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:52,801 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:52,801 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=12, startTime=1731527152425; duration=0sec 2024-11-13T19:45:52,801 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:52,801 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527213004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbb3c71819ac4c9484ea8c7137bcaf1a 2024-11-13T19:45:53,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/660ac337248d4acbbae826b6a92a7079 is 50, key is test_row_0/C:col10/1731527152455/Put/seqid=0 2024-11-13T19:45:53,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742593_1769 (size=12301) 2024-11-13T19:45:53,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/660ac337248d4acbbae826b6a92a7079 2024-11-13T19:45:53,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/db7bc53ab656479f98cafab6adf48799 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799 2024-11-13T19:45:53,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799, entries=150, sequenceid=695, filesize=30.5 K 2024-11-13T19:45:53,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/bbb3c71819ac4c9484ea8c7137bcaf1a as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a 2024-11-13T19:45:53,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a, entries=150, sequenceid=695, filesize=12.0 K 2024-11-13T19:45:53,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/660ac337248d4acbbae826b6a92a7079 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079 2024-11-13T19:45:53,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079, entries=150, sequenceid=695, filesize=12.0 K 2024-11-13T19:45:53,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f989ae1f218e9e3abedc1d7da78f0f53 in 772ms, sequenceid=695, compaction requested=false 2024-11-13T19:45:53,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:53,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T19:45:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:53,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:53,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f20b063ba75d405bae99c09a4c2ca072_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:53,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742594_1770 (size=14994) 2024-11-13T19:45:53,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:53,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f20b063ba75d405bae99c09a4c2ca072_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f20b063ba75d405bae99c09a4c2ca072_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:53,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/e8b499163e454ccdb442b91f4cffe9eb, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:53,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/e8b499163e454ccdb442b91f4cffe9eb is 175, key is test_row_0/A:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527213389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527213391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527213392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742595_1771 (size=39949) 2024-11-13T19:45:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527213504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527213504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527213504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,688 INFO [Thread-2842 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2842. 2024-11-13T19:45:53,689 DEBUG [Thread-2842 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:53,689 DEBUG [Thread-2842 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,689 DEBUG [Thread-2842 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:53,689 INFO [Thread-2840 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2840. 2024-11-13T19:45:53,689 DEBUG [Thread-2840 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:53,689 DEBUG [Thread-2840 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,690 DEBUG [Thread-2840 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:53,690 INFO [Thread-2844 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2844. 2024-11-13T19:45:53,690 DEBUG [Thread-2844 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:53,690 DEBUG [Thread-2844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:53,690 DEBUG [Thread-2844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,694 INFO [Thread-2838 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2838. 2024-11-13T19:45:53,694 DEBUG [Thread-2838 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:53,694 DEBUG [Thread-2838 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,694 DEBUG [Thread-2838 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,695 INFO [Thread-2846 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2846. 2024-11-13T19:45:53,695 DEBUG [Thread-2846 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:53,695 DEBUG [Thread-2846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,695 DEBUG [Thread-2846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:53,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:53,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37780 deadline: 1731527213717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37706 deadline: 1731527213718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:45:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37710 deadline: 1731527213717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 2024-11-13T19:45:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 , the old value is region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:45:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., hostname=54d0784d1943,36227,1731526989835, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:45:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:45:53,764 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:53,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=714, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/e8b499163e454ccdb442b91f4cffe9eb 2024-11-13T19:45:53,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/de18084eeea94fa5ac764265717f0c8d is 50, key is test_row_0/B:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:53,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742596_1772 (size=12301) 2024-11-13T19:45:53,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/de18084eeea94fa5ac764265717f0c8d 2024-11-13T19:45:53,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b3b36bf459234601b3566ce766a2bcaa is 50, key is test_row_0/C:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:53,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742597_1773 (size=12301) 2024-11-13T19:45:53,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b3b36bf459234601b3566ce766a2bcaa 2024-11-13T19:45:53,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/e8b499163e454ccdb442b91f4cffe9eb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb 2024-11-13T19:45:53,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb, entries=200, sequenceid=714, filesize=39.0 K 2024-11-13T19:45:53,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/de18084eeea94fa5ac764265717f0c8d as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d 2024-11-13T19:45:53,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d, entries=150, sequenceid=714, filesize=12.0 K 2024-11-13T19:45:53,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/b3b36bf459234601b3566ce766a2bcaa as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa 2024-11-13T19:45:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa, entries=150, sequenceid=714, filesize=12.0 K 2024-11-13T19:45:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f989ae1f218e9e3abedc1d7da78f0f53 in 635ms, sequenceid=714, compaction requested=true 2024-11-13T19:45:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:53,953 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:45:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:53,954 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:45:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f989ae1f218e9e3abedc1d7da78f0f53:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:45:53,955 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:53,955 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/A is initiating minor compaction (all files) 2024-11-13T19:45:53,955 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/A in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:53,955 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=101.0 K 2024-11-13T19:45:53,955 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:53,955 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. files: [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb] 2024-11-13T19:45:53,956 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:53,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7f99b818dd460c9c77174f0b222751, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:53,956 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/B is initiating minor compaction (all files) 2024-11-13T19:45:53,956 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/B in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:53,956 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3225baf3b76c4da2a5cf175c8b7c695c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=37.0 K 2024-11-13T19:45:53,956 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7bc53ab656479f98cafab6adf48799, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731527152278 2024-11-13T19:45:53,956 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 3225baf3b76c4da2a5cf175c8b7c695c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:53,957 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b499163e454ccdb442b91f4cffe9eb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731527152609 2024-11-13T19:45:53,957 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb3c71819ac4c9484ea8c7137bcaf1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731527152278 2024-11-13T19:45:53,957 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting de18084eeea94fa5ac764265717f0c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731527152626 2024-11-13T19:45:53,964 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:53,966 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#B#compaction#656 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:53,966 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f33b9a11ae814fc2898624dbd080b555 is 50, key is test_row_0/B:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:53,982 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111325f4b3a05755403383513a964a6a40ce_f989ae1f218e9e3abedc1d7da78f0f53 store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:53,987 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111325f4b3a05755403383513a964a6a40ce_f989ae1f218e9e3abedc1d7da78f0f53, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:53,988 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111325f4b3a05755403383513a964a6a40ce_f989ae1f218e9e3abedc1d7da78f0f53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742598_1774 (size=13391) 2024-11-13T19:45:54,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742599_1775 (size=4469) 2024-11-13T19:45:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36227 {}] regionserver.HRegion(8855): Flush requested on f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:54,034 INFO [Thread-2829 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2829. 2024-11-13T19:45:54,034 DEBUG [Thread-2829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:54,034 DEBUG [Thread-2829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,034 DEBUG [Thread-2829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,035 INFO [Thread-2831 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2831. 2024-11-13T19:45:54,035 DEBUG [Thread-2831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:54,035 DEBUG [Thread-2831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,035 DEBUG [Thread-2831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:54,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:54,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:54,041 INFO [Thread-2827 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2827. 2024-11-13T19:45:54,041 DEBUG [Thread-2827 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:54,041 DEBUG [Thread-2827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,042 DEBUG [Thread-2827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:54,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:54,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111375f276824eaf4c03893a205d1a24da4b_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527153377/Put/seqid=0 2024-11-13T19:45:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742600_1776 (size=12454) 2024-11-13T19:45:54,419 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#A#compaction#657 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:54,420 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/199daf38afd642cb805349ec90d24af7 is 175, key is test_row_0/A:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:54,424 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/f33b9a11ae814fc2898624dbd080b555 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f33b9a11ae814fc2898624dbd080b555 2024-11-13T19:45:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742601_1777 (size=32345) 2024-11-13T19:45:54,430 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/B of f989ae1f218e9e3abedc1d7da78f0f53 into f33b9a11ae814fc2898624dbd080b555(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:54,430 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:54,430 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/B, priority=13, startTime=1731527153954; duration=0sec 2024-11-13T19:45:54,430 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:45:54,430 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:B 2024-11-13T19:45:54,430 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:45:54,431 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:45:54,431 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1541): f989ae1f218e9e3abedc1d7da78f0f53/C is initiating minor compaction (all files) 2024-11-13T19:45:54,431 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f989ae1f218e9e3abedc1d7da78f0f53/C in TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:54,431 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e43e77b8125046728809c6d54afbef54, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa] into tmpdir=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp, totalSize=37.0 K 2024-11-13T19:45:54,431 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting e43e77b8125046728809c6d54afbef54, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731527151150 2024-11-13T19:45:54,432 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting 660ac337248d4acbbae826b6a92a7079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1731527152278 2024-11-13T19:45:54,432 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b36bf459234601b3566ce766a2bcaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731527152626 2024-11-13T19:45:54,452 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f989ae1f218e9e3abedc1d7da78f0f53#C#compaction#659 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-13T19:45:54,453 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/928eb1c2ef834d5e925672dc120cf91c is 50, key is test_row_0/C:col10/1731527153316/Put/seqid=0 2024-11-13T19:45:54,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742602_1778 (size=13357) 2024-11-13T19:45:54,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:54,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111375f276824eaf4c03893a205d1a24da4b_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111375f276824eaf4c03893a205d1a24da4b_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:54,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/eb1e33113de64011acbe0105b12109f9, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:54,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/eb1e33113de64011acbe0105b12109f9 is 175, key is test_row_0/A:col10/1731527153377/Put/seqid=0 2024-11-13T19:45:54,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742603_1779 (size=31255) 2024-11-13T19:45:54,834 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/199daf38afd642cb805349ec90d24af7 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/199daf38afd642cb805349ec90d24af7 2024-11-13T19:45:54,839 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/A of f989ae1f218e9e3abedc1d7da78f0f53 into 199daf38afd642cb805349ec90d24af7(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:54,839 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:54,839 INFO [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/A, priority=13, startTime=1731527153953; duration=0sec 2024-11-13T19:45:54,839 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:54,839 DEBUG [RS:0;54d0784d1943:36227-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:A 2024-11-13T19:45:54,862 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/928eb1c2ef834d5e925672dc120cf91c as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/928eb1c2ef834d5e925672dc120cf91c 2024-11-13T19:45:54,867 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f989ae1f218e9e3abedc1d7da78f0f53/C of f989ae1f218e9e3abedc1d7da78f0f53 into 928eb1c2ef834d5e925672dc120cf91c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:45:54,867 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:54,867 INFO [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., storeName=f989ae1f218e9e3abedc1d7da78f0f53/C, priority=13, startTime=1731527153954; duration=0sec 2024-11-13T19:45:54,867 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:45:54,867 DEBUG [RS:0;54d0784d1943:36227-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f989ae1f218e9e3abedc1d7da78f0f53:C 2024-11-13T19:45:54,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=733, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/eb1e33113de64011acbe0105b12109f9 2024-11-13T19:45:54,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/a20c7c6a76fe4143b8cb6c322fdc8589 is 50, key is test_row_0/B:col10/1731527153377/Put/seqid=0 2024-11-13T19:45:54,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742604_1780 (size=12301) 2024-11-13T19:45:54,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=733 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/a20c7c6a76fe4143b8cb6c322fdc8589 2024-11-13T19:45:54,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/78674503f6d84312934159a3282f6e92 is 50, key is test_row_0/C:col10/1731527153377/Put/seqid=0 2024-11-13T19:45:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742605_1781 (size=12301) 2024-11-13T19:45:54,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=733 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/78674503f6d84312934159a3282f6e92 2024-11-13T19:45:54,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/eb1e33113de64011acbe0105b12109f9 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/eb1e33113de64011acbe0105b12109f9 2024-11-13T19:45:54,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/eb1e33113de64011acbe0105b12109f9, entries=150, sequenceid=733, filesize=30.5 K 2024-11-13T19:45:54,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/a20c7c6a76fe4143b8cb6c322fdc8589 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/a20c7c6a76fe4143b8cb6c322fdc8589 2024-11-13T19:45:54,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/a20c7c6a76fe4143b8cb6c322fdc8589, entries=150, sequenceid=733, filesize=12.0 K 2024-11-13T19:45:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/78674503f6d84312934159a3282f6e92 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/78674503f6d84312934159a3282f6e92 2024-11-13T19:45:54,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/78674503f6d84312934159a3282f6e92, entries=150, sequenceid=733, filesize=12.0 K 2024-11-13T19:45:54,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for f989ae1f218e9e3abedc1d7da78f0f53 in 940ms, sequenceid=733, compaction requested=false 2024-11-13T19:45:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f989ae1f218e9e3abedc1d7da78f0f53: 2024-11-13T19:45:56,105 INFO [Thread-2835 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2835. 2024-11-13T19:45:56,106 DEBUG [Thread-2835 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:56,106 DEBUG [Thread-2835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,106 DEBUG [Thread-2835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:56,116 INFO [Thread-2833 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2833. 2024-11-13T19:45:56,116 DEBUG [Thread-2833 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:45:56,116 DEBUG [Thread-2833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,116 DEBUG [Thread-2833 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 194 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2073 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1942 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1909 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2076 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1910 2024-11-13T19:45:56,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:45:56,117 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:45:56,117 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:56,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:56,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:56,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:45:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:56,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527156122"}]},"ts":"1731527156122"} 2024-11-13T19:45:56,124 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:45:56,124 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:45:56,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:45:56,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, UNASSIGN}] 2024-11-13T19:45:56,127 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, UNASSIGN 2024-11-13T19:45:56,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=CLOSING, regionLocation=54d0784d1943,36227,1731526989835 2024-11-13T19:45:56,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, UNASSIGN because future has completed 2024-11-13T19:45:56,130 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:45:56,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835}] 2024-11-13T19:45:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:56,285 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing f989ae1f218e9e3abedc1d7da78f0f53, disabling compactions & flushes 2024-11-13T19:45:56,286 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. after waiting 0 ms 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:56,286 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing f989ae1f218e9e3abedc1d7da78f0f53 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=A 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=B 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f989ae1f218e9e3abedc1d7da78f0f53, store=C 2024-11-13T19:45:56,286 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T19:45:56,296 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134e18165418f04e50999d5e47cb6ff9a2_f989ae1f218e9e3abedc1d7da78f0f53 is 50, key is test_row_0/A:col10/1731527156114/Put/seqid=0 2024-11-13T19:45:56,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742606_1782 (size=9914) 2024-11-13T19:45:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:56,715 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:45:56,718 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134e18165418f04e50999d5e47cb6ff9a2_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e18165418f04e50999d5e47cb6ff9a2_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:56,719 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/bc64d35c4ba549b2b93e8951ec3a4ce5, store: [table=TestAcidGuarantees family=A region=f989ae1f218e9e3abedc1d7da78f0f53] 2024-11-13T19:45:56,720 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/bc64d35c4ba549b2b93e8951ec3a4ce5 is 175, key is test_row_0/A:col10/1731527156114/Put/seqid=0 2024-11-13T19:45:56,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742607_1783 (size=22561) 2024-11-13T19:45:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:57,124 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=741, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/bc64d35c4ba549b2b93e8951ec3a4ce5 2024-11-13T19:45:57,139 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/fb78bf261bab43ff866c1163b412c8d8 is 50, key is test_row_0/B:col10/1731527156114/Put/seqid=0 2024-11-13T19:45:57,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742608_1784 (size=9857) 2024-11-13T19:45:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:57,564 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/fb78bf261bab43ff866c1163b412c8d8 2024-11-13T19:45:57,579 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/521650f6b4bd4fd0b29bbcbfee7efca0 is 50, key is test_row_0/C:col10/1731527156114/Put/seqid=0 2024-11-13T19:45:57,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742609_1785 (size=9857) 2024-11-13T19:45:58,022 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=741 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/521650f6b4bd4fd0b29bbcbfee7efca0 2024-11-13T19:45:58,026 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/A/bc64d35c4ba549b2b93e8951ec3a4ce5 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/bc64d35c4ba549b2b93e8951ec3a4ce5 2024-11-13T19:45:58,031 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/bc64d35c4ba549b2b93e8951ec3a4ce5, entries=100, sequenceid=741, filesize=22.0 K 2024-11-13T19:45:58,032 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/B/fb78bf261bab43ff866c1163b412c8d8 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/fb78bf261bab43ff866c1163b412c8d8 2024-11-13T19:45:58,039 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/fb78bf261bab43ff866c1163b412c8d8, entries=100, sequenceid=741, filesize=9.6 K 2024-11-13T19:45:58,040 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/.tmp/C/521650f6b4bd4fd0b29bbcbfee7efca0 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/521650f6b4bd4fd0b29bbcbfee7efca0 2024-11-13T19:45:58,044 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/521650f6b4bd4fd0b29bbcbfee7efca0, entries=100, sequenceid=741, filesize=9.6 K 2024-11-13T19:45:58,044 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f989ae1f218e9e3abedc1d7da78f0f53 in 1758ms, sequenceid=741, compaction requested=true 2024-11-13T19:45:58,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb] to archive 2024-11-13T19:45:58,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:58,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/76b6130cb5db47cfa1308e3ff7b99f17 2024-11-13T19:45:58,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c3f870949570413f80629bccf23defb8 2024-11-13T19:45:58,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/3db6c5339bd14bf4b134f658f6ac0495 2024-11-13T19:45:58,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6f556f95a3844c65b2b31cfcf0580085 2024-11-13T19:45:58,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/de894cb7cd114fb9bb4bee277dac89f5 2024-11-13T19:45:58,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/c69115c771d44738be8d0d4f2a328ff6 2024-11-13T19:45:58,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/584835d77e804d3e8a73afea3eed4957 2024-11-13T19:45:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dafbf1b665e442ef8d91c814c00640f4 2024-11-13T19:45:58,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a989085f0d524e6da902ca83c1e90ccb 2024-11-13T19:45:58,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2b277a48e7ab4a67b7eae01189c0ad10 2024-11-13T19:45:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/4a7a5348390045cba82a0e3999cd0424 2024-11-13T19:45:58,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/fe0042df63a748b7897fb5733b44142e 2024-11-13T19:45:58,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/dfd4e29821a24a859e84b61c7d1f6eb8 2024-11-13T19:45:58,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6b33f2f65d5b4a3594d80d6c28e0eb56 2024-11-13T19:45:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/26684e49fae0491fbe4b777b8f02a6e4 2024-11-13T19:45:58,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/2bd1ee1df2ad4456b0a4465795e14259 2024-11-13T19:45:58,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/744c9d92385e489b8ecaaf288f809e01 2024-11-13T19:45:58,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/7e01d37ea91d4442af9a82fa06ed0cc7 2024-11-13T19:45:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ce0061884f5b4d959987cae60d73585b 2024-11-13T19:45:58,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d1e6ab3f1f1d4a029faca001a3f0f848 2024-11-13T19:45:58,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d0f369ceb80e4e9b891c47a1f1e4bef4 2024-11-13T19:45:58,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ef6e966b27b14d23885e340b1df55193 2024-11-13T19:45:58,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/ba5fb5ff0ff142eda9f2e3c727f966fc 2024-11-13T19:45:58,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/0691ce4fc1074c2d821d8b6197eb7ab7 2024-11-13T19:45:58,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/d608478ba2fd4410b838e62faacc672a 2024-11-13T19:45:58,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/aec080e0ad004a81b95386547ae312fa 2024-11-13T19:45:58,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/72423baf3e0b49b0a1ec79cb97d8b110 2024-11-13T19:45:58,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/9c20214141da46859dd0f570a915744b 2024-11-13T19:45:58,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/a5a4f51b41914e51bd5cb4f47e1c162a 2024-11-13T19:45:58,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/6c7f99b818dd460c9c77174f0b222751 2024-11-13T19:45:58,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/db7bc53ab656479f98cafab6adf48799 2024-11-13T19:45:58,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/e8b499163e454ccdb442b91f4cffe9eb 2024-11-13T19:45:58,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/65f92892d4ae4a4c8d2f89455561a4ce, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9ed5eff52fa14d578fb53fc9a846b5b5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e7ee1769af6a4605ba8d322d73e2e960, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3777c3901502483182389c19f9ac7b15, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/947d83f612db46c796e3db9e1d5b8849, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/95cb9fdab1ad45a2b463c3bf5ddfd47b, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9c375a8a4b40437780588ca4835503f3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3225baf3b76c4da2a5cf175c8b7c695c, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d] to archive 2024-11-13T19:45:58,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:58,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/76c0b8f1dbb449f885472ad80cde945d 2024-11-13T19:45:58,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/15277fd8250b443d8646a35b83def81c 2024-11-13T19:45:58,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/38afc8c5b1e8466ca11661f79b187577 2024-11-13T19:45:58,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/65f92892d4ae4a4c8d2f89455561a4ce to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/65f92892d4ae4a4c8d2f89455561a4ce 2024-11-13T19:45:58,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/99af8857a7a74fc39d34988a8f2bf03b 2024-11-13T19:45:58,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/0612bbb4b31344d79bfcbe1c94096fa1 2024-11-13T19:45:58,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e933d937735743ca885268bd354d936d 2024-11-13T19:45:58,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/01e62a4358604b649cfba5e981dfe35a 2024-11-13T19:45:58,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9ed5eff52fa14d578fb53fc9a846b5b5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9ed5eff52fa14d578fb53fc9a846b5b5 2024-11-13T19:45:58,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/58bbfce3b50a4802bfc911f98a848d73 2024-11-13T19:45:58,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b319642d99a141ae81df05578826685f 2024-11-13T19:45:58,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e7ee1769af6a4605ba8d322d73e2e960 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/e7ee1769af6a4605ba8d322d73e2e960 2024-11-13T19:45:58,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/67fff0635e2642799bef2aeb0c0a316d 2024-11-13T19:45:58,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3d60052f8466456a99ca391107ac2d18 2024-11-13T19:45:58,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/23500622016b425a8738736fc3e19038 2024-11-13T19:45:58,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3777c3901502483182389c19f9ac7b15 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3777c3901502483182389c19f9ac7b15 2024-11-13T19:45:58,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f301157dd51e4b209922f586607e042a 2024-11-13T19:45:58,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/49f950b7682c46f5bd09444bbb76b788 2024-11-13T19:45:58,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/947d83f612db46c796e3db9e1d5b8849 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/947d83f612db46c796e3db9e1d5b8849 2024-11-13T19:45:58,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8724cb99e95a408eb5acd3a6d55f6616 2024-11-13T19:45:58,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f869369a7159457c8073e7092c6a8c8a 2024-11-13T19:45:58,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/95cb9fdab1ad45a2b463c3bf5ddfd47b to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/95cb9fdab1ad45a2b463c3bf5ddfd47b 2024-11-13T19:45:58,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/4c46e2fd97514567b13a6ee06f1961ba 2024-11-13T19:45:58,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbfdbdc3b6264448a224008c6179d474 2024-11-13T19:45:58,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9c375a8a4b40437780588ca4835503f3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/9c375a8a4b40437780588ca4835503f3 2024-11-13T19:45:58,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/8cf8736e24d040a791338f63cf109ced 2024-11-13T19:45:58,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/2af0230a4df54b04a20960cfaa0c086c 2024-11-13T19:45:58,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/c94400b2e3cc4c27a7ae0df163ae7c83 2024-11-13T19:45:58,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3225baf3b76c4da2a5cf175c8b7c695c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/3225baf3b76c4da2a5cf175c8b7c695c 2024-11-13T19:45:58,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/b952f03bc844456fb51428d91765750f 2024-11-13T19:45:58,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/bbb3c71819ac4c9484ea8c7137bcaf1a 2024-11-13T19:45:58,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/de18084eeea94fa5ac764265717f0c8d 2024-11-13T19:45:58,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/574e562bcb5b461d8a9cb557cb70aa46, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/4d5dba2eaa5d447ca3c0ed3a72b1439f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/96a64e20952e401b8c98a0133a9a31f6, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/c248b967896f4388b4dc25c69626f9fe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/08af832b8aab47cab71e07dee027131f, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/58f0144c1ef042338d1fffdb6f01d872, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e43e77b8125046728809c6d54afbef54, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa] to archive 2024-11-13T19:45:58,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:45:58,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/3eb9674b711841e28d8e5201d1e806b4 2024-11-13T19:45:58,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/ac794e60faea495098960e56c972f5b1 2024-11-13T19:45:58,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/9bf8859ebf7d4959910ab6046af1aca7 2024-11-13T19:45:58,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/574e562bcb5b461d8a9cb557cb70aa46 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/574e562bcb5b461d8a9cb557cb70aa46 2024-11-13T19:45:58,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a81d5b41e4484998a65a842e9e04dc5d 2024-11-13T19:45:58,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/34a3c9b00b7f4999a23f5ad9982ef444 2024-11-13T19:45:58,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/91258d0149934acb8fce16f3ead67d05 2024-11-13T19:45:58,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/81d6bbf479714c8e9461e5588ffd8a47 2024-11-13T19:45:58,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/4d5dba2eaa5d447ca3c0ed3a72b1439f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/4d5dba2eaa5d447ca3c0ed3a72b1439f 2024-11-13T19:45:58,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e0cafc1e93844abba1702ee0101fb3e3 2024-11-13T19:45:58,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a475978aabc14020b187dbc346739749 2024-11-13T19:45:58,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/96a64e20952e401b8c98a0133a9a31f6 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/96a64e20952e401b8c98a0133a9a31f6 2024-11-13T19:45:58,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b4d5e710b60b499c900c5ffb25bbaadc 2024-11-13T19:45:58,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2e56afac64aa4f0e970175e0fa7ddcf8 2024-11-13T19:45:58,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/85209c9d7d1147089084517e4bf68e99 2024-11-13T19:45:58,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/c248b967896f4388b4dc25c69626f9fe to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/c248b967896f4388b4dc25c69626f9fe 2024-11-13T19:45:58,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/5b30a5e8bb5d42a7baa3e2a4bb69cf96 2024-11-13T19:45:58,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0732d0ca834147a887c4bcb013b19935 2024-11-13T19:45:58,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/35150b0f514449d6a08544ffd999215e 2024-11-13T19:45:58,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/08af832b8aab47cab71e07dee027131f to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/08af832b8aab47cab71e07dee027131f 2024-11-13T19:45:58,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b7a6a774cf5a43b5aab222c6aeb88aaf 2024-11-13T19:45:58,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/a911420c6ca24f16b84233e482a4babe 2024-11-13T19:45:58,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/2ae3bb6c45b54ecc9d26e45b783ae9e8 2024-11-13T19:45:58,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/58f0144c1ef042338d1fffdb6f01d872 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/58f0144c1ef042338d1fffdb6f01d872 2024-11-13T19:45:58,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/0b36d8ff30594e3186ce3b81d589000d 2024-11-13T19:45:58,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/92190d22b404404faef1e237d0f5f5e5 2024-11-13T19:45:58,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b9072b4ef16b4cb78c54c785444a0872 2024-11-13T19:45:58,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e43e77b8125046728809c6d54afbef54 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/e43e77b8125046728809c6d54afbef54 2024-11-13T19:45:58,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/d0d75927a51d4a5aae1e41165f0f2ae4 2024-11-13T19:45:58,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/660ac337248d4acbbae826b6a92a7079 2024-11-13T19:45:58,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/b3b36bf459234601b3566ce766a2bcaa 2024-11-13T19:45:58,205 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits/744.seqid, newMaxSeqId=744, maxSeqId=4 2024-11-13T19:45:58,205 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. 2024-11-13T19:45:58,205 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for f989ae1f218e9e3abedc1d7da78f0f53: Waiting for close lock at 1731527156286Running coprocessor pre-close hooks at 1731527156286Disabling compacts and flushes for region at 1731527156286Disabling writes for close at 1731527156286Obtaining lock to block concurrent updates at 1731527156286Preparing flush snapshotting stores in f989ae1f218e9e3abedc1d7da78f0f53 at 1731527156286Finished memstore snapshotting TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731527156286Flushing stores of TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53. at 1731527156289 (+3 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/A: creating writer at 1731527156289Flushing mob file f989ae1f218e9e3abedc1d7da78f0f53/A: appending metadata at 1731527156296 (+7 ms)Flushing mob file f989ae1f218e9e3abedc1d7da78f0f53/A: closing flushed file at 1731527156296Flushing f989ae1f218e9e3abedc1d7da78f0f53/A: appending metadata at 1731527156719 (+423 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/A: closing flushed file at 1731527156719Flushing f989ae1f218e9e3abedc1d7da78f0f53/B: creating writer at 1731527157130 (+411 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/B: appending metadata at 1731527157138 (+8 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/B: closing flushed file at 1731527157138Flushing f989ae1f218e9e3abedc1d7da78f0f53/C: creating writer at 1731527157574 (+436 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/C: appending metadata at 1731527157578 (+4 ms)Flushing f989ae1f218e9e3abedc1d7da78f0f53/C: closing flushed file at 1731527157578Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@647cb45f: reopening flushed file at 1731527158026 (+448 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@285552e0: reopening flushed file at 1731527158031 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c81e8b2: reopening flushed file at 1731527158039 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f989ae1f218e9e3abedc1d7da78f0f53 in 1758ms, sequenceid=741, compaction requested=true at 1731527158044 (+5 ms)Writing region close event to WAL at 1731527158201 (+157 ms)Running coprocessor post-close hooks at 1731527158205 (+4 ms)Closed at 1731527158205 2024-11-13T19:45:58,207 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=f989ae1f218e9e3abedc1d7da78f0f53, regionState=CLOSED 2024-11-13T19:45:58,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 because future has completed 2024-11-13T19:45:58,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T19:45:58,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure f989ae1f218e9e3abedc1d7da78f0f53, server=54d0784d1943,36227,1731526989835 in 2.0820 sec 2024-11-13T19:45:58,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-13T19:45:58,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f989ae1f218e9e3abedc1d7da78f0f53, UNASSIGN in 2.0890 sec 2024-11-13T19:45:58,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T19:45:58,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0920 sec 2024-11-13T19:45:58,220 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527158220"}]},"ts":"1731527158220"} 2024-11-13T19:45:58,221 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:45:58,221 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:45:58,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1040 sec 2024-11-13T19:45:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:45:58,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:58,275 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:45:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,277 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T19:45:58,278 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,306 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C, FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits] 2024-11-13T19:45:58,309 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/199daf38afd642cb805349ec90d24af7 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/199daf38afd642cb805349ec90d24af7 2024-11-13T19:45:58,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/bc64d35c4ba549b2b93e8951ec3a4ce5 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/bc64d35c4ba549b2b93e8951ec3a4ce5 2024-11-13T19:45:58,313 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/eb1e33113de64011acbe0105b12109f9 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/A/eb1e33113de64011acbe0105b12109f9 2024-11-13T19:45:58,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/a20c7c6a76fe4143b8cb6c322fdc8589 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/a20c7c6a76fe4143b8cb6c322fdc8589 2024-11-13T19:45:58,319 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f33b9a11ae814fc2898624dbd080b555 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/f33b9a11ae814fc2898624dbd080b555 2024-11-13T19:45:58,320 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/fb78bf261bab43ff866c1163b412c8d8 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/B/fb78bf261bab43ff866c1163b412c8d8 2024-11-13T19:45:58,324 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/521650f6b4bd4fd0b29bbcbfee7efca0 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/521650f6b4bd4fd0b29bbcbfee7efca0 2024-11-13T19:45:58,326 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/78674503f6d84312934159a3282f6e92 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/78674503f6d84312934159a3282f6e92 2024-11-13T19:45:58,328 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/928eb1c2ef834d5e925672dc120cf91c to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/C/928eb1c2ef834d5e925672dc120cf91c 2024-11-13T19:45:58,331 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits/744.seqid to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53/recovered.edits/744.seqid 2024-11-13T19:45:58,332 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/default/TestAcidGuarantees/f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,332 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:45:58,333 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:45:58,334 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:45:58,347 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131003f75662914932a28a3babd331fbcc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131003f75662914932a28a3babd331fbcc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131507771218264777bab62dbedd6e0735_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131507771218264777bab62dbedd6e0735_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,350 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113273e8e5f2c5b41329ce6ecb0d43aaa54_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113273e8e5f2c5b41329ce6ecb0d43aaa54_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329008eed2777456895388b681da83a59_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111329008eed2777456895388b681da83a59_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,353 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113390faee0327b4dafa7c5733928067643_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113390faee0327b4dafa7c5733928067643_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,355 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113473d5682d5be4a249412f0306b638b39_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113473d5682d5be4a249412f0306b638b39_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,356 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6d0f2fd73e4fc3b6d83fbe2a41dd05_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134a6d0f2fd73e4fc3b6d83fbe2a41dd05_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,357 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e18165418f04e50999d5e47cb6ff9a2_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e18165418f04e50999d5e47cb6ff9a2_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113605de99a26b047379dcc1bf2b74766ca_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113605de99a26b047379dcc1bf2b74766ca_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,362 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362aeb0785b144c38bc3cf3c191325feb_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111362aeb0785b144c38bc3cf3c191325feb_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,364 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113690a173f268b4f2981f88adfcc5d9455_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113690a173f268b4f2981f88adfcc5d9455_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,365 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111375f276824eaf4c03893a205d1a24da4b_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111375f276824eaf4c03893a205d1a24da4b_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,367 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376b0eba736ee47fe94be0f9308216770_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376b0eba736ee47fe94be0f9308216770_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,368 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b4f9d860ef4446fa61ca3dc6397faab_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b4f9d860ef4446fa61ca3dc6397faab_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,369 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139085bf633d8f4057a87fc2e1c92feb5b_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139085bf633d8f4057a87fc2e1c92feb5b_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,371 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b55c579d45714bdb82c5bd56c3d0afdd_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b55c579d45714bdb82c5bd56c3d0afdd_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,372 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb128477927a4a3a8be6f0588340f39e_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb128477927a4a3a8be6f0588340f39e_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,376 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3a650e7ff8b4c5bb673024daef3ceef_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c3a650e7ff8b4c5bb673024daef3ceef_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,377 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cd57720eec504b27a43e0e80b9330e1f_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cd57720eec504b27a43e0e80b9330e1f_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,378 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8c878333d3f4998a43d6012810252bc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8c878333d3f4998a43d6012810252bc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,379 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d94523416f0f4b769e040be12edabba8_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d94523416f0f4b769e040be12edabba8_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,381 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9a605a6a3f742f1b7fc34a40a7d87dc_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9a605a6a3f742f1b7fc34a40a7d87dc_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,383 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f20b063ba75d405bae99c09a4c2ca072_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f20b063ba75d405bae99c09a4c2ca072_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T19:45:58,384 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f5bc815453ab49b6ac7ffe50f9efb742_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f5bc815453ab49b6ac7ffe50f9efb742_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,386 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f650d94ad16e4d3bb94ee6967d1fede9_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f650d94ad16e4d3bb94ee6967d1fede9_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,388 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7b4db2ab59d44ec8ac9d0dd7778d624_f989ae1f218e9e3abedc1d7da78f0f53 to hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7b4db2ab59d44ec8ac9d0dd7778d624_f989ae1f218e9e3abedc1d7da78f0f53 2024-11-13T19:45:58,388 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:45:58,391 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,394 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:45:58,396 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:45:58,397 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,397 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:45:58,398 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731527158397"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:58,400 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:45:58,400 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f989ae1f218e9e3abedc1d7da78f0f53, NAME => 'TestAcidGuarantees,,1731527132484.f989ae1f218e9e3abedc1d7da78f0f53.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:45:58,400 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:45:58,400 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731527158400"}]},"ts":"9223372036854775807"} 2024-11-13T19:45:58,402 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:45:58,402 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:45:58,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 127 msec 2024-11-13T19:45:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T19:45:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:45:58,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:45:58,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 252), OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=840 (was 776) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=953 (was 1503) 2024-11-13T19:45:58,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T19:45:58,606 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:45:58,606 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:58,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:58,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:58,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:45:58,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T19:45:58,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1570076000, stopped=false 2024-11-13T19:45:58,607 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=54d0784d1943,42037,1731526988795 2024-11-13T19:45:58,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:45:58,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:45:58,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:45:58,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:45:58,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:45:58,611 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:45:58,611 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:58,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:58,612 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '54d0784d1943,36227,1731526989835' ***** 2024-11-13T19:45:58,612 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T19:45:58,612 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T19:45:58,613 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(959): stopping server 54d0784d1943,36227,1731526989835 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;54d0784d1943:36227. 2024-11-13T19:45:58,613 DEBUG [RS:0;54d0784d1943:36227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:45:58,613 DEBUG [RS:0;54d0784d1943:36227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T19:45:58,613 INFO [RS:0;54d0784d1943:36227 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T19:45:58,614 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T19:45:58,614 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T19:45:58,614 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T19:45:58,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:45:58,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:45:58,615 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:45:58,615 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:45:58,615 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:45:58,615 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:45:58,615 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T19:45:58,615 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:45:58,615 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T19:45:58,640 INFO [regionserver/54d0784d1943:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:45:58,652 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/info/655c8cb77baf4d0cbbf8d70c575234b2 is 95, key is TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39./info:/1731527018797/DeleteFamily/seqid=0 2024-11-13T19:45:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742610_1786 (size=6514) 2024-11-13T19:45:58,657 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/info/655c8cb77baf4d0cbbf8d70c575234b2 2024-11-13T19:45:58,680 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/ns/3e5d790ddbb94dbbb69fc104bdb27dea is 93, key is TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39./ns:/1731527018747/DeleteFamily/seqid=0 2024-11-13T19:45:58,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742611_1787 (size=5906) 2024-11-13T19:45:58,815 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:45:59,014 INFO [regionserver/54d0784d1943:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T19:45:59,014 INFO [regionserver/54d0784d1943:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T19:45:59,015 DEBUG [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:45:59,084 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/ns/3e5d790ddbb94dbbb69fc104bdb27dea 2024-11-13T19:45:59,104 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/rep_barrier/55be04df640a4e3097bc62da1a904f64 is 102, key is TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39./rep_barrier:/1731527018747/DeleteFamily/seqid=0 2024-11-13T19:45:59,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742612_1788 (size=6025) 2024-11-13T19:45:59,108 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/rep_barrier/55be04df640a4e3097bc62da1a904f64 2024-11-13T19:45:59,135 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/table/92599da35d34442bb52139d6137566cb is 96, key is TestAcidGuarantees,,1731526994387.31328a0914442b3583375df8a5770e39./table:/1731527018747/DeleteFamily/seqid=0 2024-11-13T19:45:59,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742613_1789 (size=5936) 2024-11-13T19:45:59,156 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/table/92599da35d34442bb52139d6137566cb 2024-11-13T19:45:59,160 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/info/655c8cb77baf4d0cbbf8d70c575234b2 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/info/655c8cb77baf4d0cbbf8d70c575234b2 2024-11-13T19:45:59,163 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/info/655c8cb77baf4d0cbbf8d70c575234b2, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T19:45:59,164 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/ns/3e5d790ddbb94dbbb69fc104bdb27dea as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/ns/3e5d790ddbb94dbbb69fc104bdb27dea 2024-11-13T19:45:59,168 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/ns/3e5d790ddbb94dbbb69fc104bdb27dea, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T19:45:59,168 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/rep_barrier/55be04df640a4e3097bc62da1a904f64 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/rep_barrier/55be04df640a4e3097bc62da1a904f64 2024-11-13T19:45:59,172 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/rep_barrier/55be04df640a4e3097bc62da1a904f64, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T19:45:59,173 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/.tmp/table/92599da35d34442bb52139d6137566cb as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/table/92599da35d34442bb52139d6137566cb 2024-11-13T19:45:59,180 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/table/92599da35d34442bb52139d6137566cb, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T19:45:59,181 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 566ms, sequenceid=90, compaction requested=false 2024-11-13T19:45:59,186 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T19:45:59,187 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:45:59,187 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:45:59,187 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731527158614Running coprocessor pre-close hooks at 1731527158615 (+1 ms)Disabling compacts and flushes for region at 1731527158615Disabling writes for close at 1731527158615Obtaining lock to block concurrent updates at 1731527158615Preparing flush snapshotting stores in 1588230740 at 1731527158615Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731527158616 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731527158616Flushing 1588230740/info: creating writer at 1731527158617 (+1 ms)Flushing 1588230740/info: appending metadata at 1731527158652 (+35 ms)Flushing 1588230740/info: closing flushed file at 1731527158652Flushing 1588230740/ns: creating writer at 1731527158662 (+10 ms)Flushing 1588230740/ns: appending metadata at 1731527158680 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731527158680Flushing 1588230740/rep_barrier: creating writer at 1731527159088 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731527159104 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731527159104Flushing 1588230740/table: creating writer at 1731527159112 (+8 ms)Flushing 1588230740/table: appending metadata at 1731527159135 (+23 ms)Flushing 1588230740/table: closing flushed file at 1731527159135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bbfc796: reopening flushed file at 1731527159159 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b3f1363: reopening flushed file at 1731527159163 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@612d47be: reopening flushed file at 1731527159168 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30dac48a: reopening flushed file at 1731527159172 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 566ms, sequenceid=90, compaction requested=false at 1731527159181 (+9 ms)Writing region close event to WAL at 1731527159183 (+2 ms)Running coprocessor post-close hooks at 1731527159187 (+4 ms)Closed at 1731527159187 2024-11-13T19:45:59,187 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T19:45:59,216 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(976): stopping server 54d0784d1943,36227,1731526989835; all regions closed. 2024-11-13T19:45:59,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741834_1010 (size=25344) 2024-11-13T19:45:59,228 DEBUG [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/oldWALs 2024-11-13T19:45:59,228 INFO [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 54d0784d1943%2C36227%2C1731526989835.meta:.meta(num 1731526993835) 2024-11-13T19:45:59,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741833_1009 (size=30368698) 2024-11-13T19:45:59,233 DEBUG [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/oldWALs 2024-11-13T19:45:59,233 INFO [RS:0;54d0784d1943:36227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 54d0784d1943%2C36227%2C1731526989835:(num 1731526993206) 2024-11-13T19:45:59,233 DEBUG [RS:0;54d0784d1943:36227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:45:59,233 INFO [RS:0;54d0784d1943:36227 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:45:59,233 INFO [RS:0;54d0784d1943:36227 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:45:59,233 INFO [RS:0;54d0784d1943:36227 {}] hbase.ChoreService(370): Chore service for: regionserver/54d0784d1943:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T19:45:59,233 INFO [RS:0;54d0784d1943:36227 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:45:59,233 INFO [regionserver/54d0784d1943:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:45:59,234 INFO [RS:0;54d0784d1943:36227 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36227 2024-11-13T19:45:59,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/54d0784d1943,36227,1731526989835 2024-11-13T19:45:59,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:45:59,240 INFO [RS:0;54d0784d1943:36227 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:45:59,240 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [54d0784d1943,36227,1731526989835] 2024-11-13T19:45:59,244 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/54d0784d1943,36227,1731526989835 already deleted, retry=false 2024-11-13T19:45:59,244 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 54d0784d1943,36227,1731526989835 expired; onlineServers=0 2024-11-13T19:45:59,244 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '54d0784d1943,42037,1731526988795' ***** 2024-11-13T19:45:59,244 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T19:45:59,244 INFO [M:0;54d0784d1943:42037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:45:59,244 INFO [M:0;54d0784d1943:42037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:45:59,245 DEBUG [M:0;54d0784d1943:42037 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T19:45:59,245 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T19:45:59,245 DEBUG [M:0;54d0784d1943:42037 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T19:45:59,245 DEBUG [master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731526992831 {}] cleaner.HFileCleaner(306): Exit Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731526992831,5,FailOnTimeoutGroup] 2024-11-13T19:45:59,245 DEBUG [master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731526992830 {}] cleaner.HFileCleaner(306): Exit Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731526992830,5,FailOnTimeoutGroup] 2024-11-13T19:45:59,245 INFO [M:0;54d0784d1943:42037 {}] hbase.ChoreService(370): Chore service for: master/54d0784d1943:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T19:45:59,245 INFO [M:0;54d0784d1943:42037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:45:59,245 DEBUG [M:0;54d0784d1943:42037 {}] master.HMaster(1795): Stopping service threads 2024-11-13T19:45:59,245 INFO [M:0;54d0784d1943:42037 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T19:45:59,245 INFO [M:0;54d0784d1943:42037 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:45:59,245 ERROR [M:0;54d0784d1943:42037 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:34445 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34445,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T19:45:59,246 INFO [M:0;54d0784d1943:42037 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T19:45:59,247 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T19:45:59,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T19:45:59,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:45:59,247 DEBUG [M:0;54d0784d1943:42037 {}] zookeeper.ZKUtil(347): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T19:45:59,248 WARN [M:0;54d0784d1943:42037 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T19:45:59,248 INFO [M:0;54d0784d1943:42037 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/.lastflushedseqids 2024-11-13T19:45:59,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742614_1790 (size=72) 2024-11-13T19:45:59,264 INFO [M:0;54d0784d1943:42037 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T19:45:59,264 INFO [M:0;54d0784d1943:42037 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T19:45:59,264 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:45:59,264 INFO [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:45:59,264 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:45:59,264 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:45:59,264 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:45:59,264 INFO [M:0;54d0784d1943:42037 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=781.71 KB heapSize=958.48 KB 2024-11-13T19:45:59,284 DEBUG [M:0;54d0784d1943:42037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81bb36b06e9e4ee0a82eb0c0670dc374 is 82, key is hbase:meta,,1/info:regioninfo/1731526993965/Put/seqid=0 2024-11-13T19:45:59,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742615_1791 (size=5672) 2024-11-13T19:45:59,289 INFO [M:0;54d0784d1943:42037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2190 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81bb36b06e9e4ee0a82eb0c0670dc374 2024-11-13T19:45:59,317 DEBUG [M:0;54d0784d1943:42037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca85f16a96814743a1c43cea96abf7db is 2479, key is \x00\x00\x00\x00\x00\x00\x00R/proc:d/1731527080186/Put/seqid=0 2024-11-13T19:45:59,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742616_1792 (size=44306) 2024-11-13T19:45:59,322 INFO [M:0;54d0784d1943:42037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=781.15 KB at sequenceid=2190 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca85f16a96814743a1c43cea96abf7db 2024-11-13T19:45:59,327 INFO [M:0;54d0784d1943:42037 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca85f16a96814743a1c43cea96abf7db 2024-11-13T19:45:59,328 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:45:59,328 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T19:45:59,329 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:45:59,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:45:59,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36227-0x100eda004b50001, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:45:59,344 INFO [RS:0;54d0784d1943:36227 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:45:59,344 INFO [RS:0;54d0784d1943:36227 {}] regionserver.HRegionServer(1031): Exiting; stopping=54d0784d1943,36227,1731526989835; zookeeper connection closed. 2024-11-13T19:45:59,357 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@327bdfb9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@327bdfb9 2024-11-13T19:45:59,358 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T19:45:59,361 DEBUG [M:0;54d0784d1943:42037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e54ecfa12c304e4bb06846c45385ec41 is 69, key is 54d0784d1943,36227,1731526989835/rs:state/1731526992872/Put/seqid=0 2024-11-13T19:45:59,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073742617_1793 (size=5156) 2024-11-13T19:45:59,766 INFO [M:0;54d0784d1943:42037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2190 (bloomFilter=true), to=hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e54ecfa12c304e4bb06846c45385ec41 2024-11-13T19:45:59,771 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81bb36b06e9e4ee0a82eb0c0670dc374 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/81bb36b06e9e4ee0a82eb0c0670dc374 2024-11-13T19:45:59,779 INFO [M:0;54d0784d1943:42037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/81bb36b06e9e4ee0a82eb0c0670dc374, entries=8, sequenceid=2190, filesize=5.5 K 2024-11-13T19:45:59,780 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ca85f16a96814743a1c43cea96abf7db as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ca85f16a96814743a1c43cea96abf7db 2024-11-13T19:45:59,785 INFO [M:0;54d0784d1943:42037 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca85f16a96814743a1c43cea96abf7db 2024-11-13T19:45:59,785 INFO [M:0;54d0784d1943:42037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ca85f16a96814743a1c43cea96abf7db, entries=164, sequenceid=2190, filesize=43.3 K 2024-11-13T19:45:59,786 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e54ecfa12c304e4bb06846c45385ec41 as hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e54ecfa12c304e4bb06846c45385ec41 2024-11-13T19:45:59,790 INFO [M:0;54d0784d1943:42037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34445/user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e54ecfa12c304e4bb06846c45385ec41, entries=1, sequenceid=2190, filesize=5.0 K 2024-11-13T19:45:59,791 INFO [M:0;54d0784d1943:42037 {}] regionserver.HRegion(3140): Finished flush of dataSize ~781.71 KB/800468, heapSize ~958.18 KB/981176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2190, compaction requested=false 2024-11-13T19:45:59,805 INFO [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:45:59,806 DEBUG [M:0;54d0784d1943:42037 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731527159264Disabling compacts and flushes for region at 1731527159264Disabling writes for close at 1731527159264Obtaining lock to block concurrent updates at 1731527159264Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731527159264Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=800468, getHeapSize=981416, getOffHeapSize=0, getCellsCount=2379 at 1731527159265 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731527159265Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731527159265Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731527159284 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731527159284Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731527159294 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731527159316 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731527159316Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731527159327 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731527159361 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731527159361Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61bd1299: reopening flushed file at 1731527159770 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@652b0489: reopening flushed file at 1731527159779 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e81c85f: reopening flushed file at 1731527159785 (+6 ms)Finished flush of dataSize ~781.71 KB/800468, heapSize ~958.18 KB/981176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2190, compaction requested=false at 1731527159792 (+7 ms)Writing region close event to WAL at 1731527159805 (+13 ms)Closed at 1731527159805 2024-11-13T19:45:59,810 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e9992caf-b90e-3912-1d69-ae700de5ef3e/MasterData/WALs/54d0784d1943,42037,1731526988795/54d0784d1943%2C42037%2C1731526988795.1731526991916 not finished, retry = 0 2024-11-13T19:45:59,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38309 is added to blk_1073741830_1006 (size=943993) 2024-11-13T19:45:59,911 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:45:59,911 INFO [M:0;54d0784d1943:42037 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T19:45:59,911 INFO [M:0;54d0784d1943:42037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42037 2024-11-13T19:45:59,912 INFO [M:0;54d0784d1943:42037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:46:00,018 INFO [M:0;54d0784d1943:42037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:46:00,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:46:00,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42037-0x100eda004b50000, quorum=127.0.0.1:50411, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:46:00,028 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:46:00,031 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:46:00,031 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:46:00,031 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:46:00,031 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.log.dir/,STOPPED} 2024-11-13T19:46:00,037 WARN [BP-1564430831-172.17.0.2-1731526985115 heartbeating to localhost/127.0.0.1:34445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T19:46:00,037 WARN [BP-1564430831-172.17.0.2-1731526985115 heartbeating to localhost/127.0.0.1:34445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1564430831-172.17.0.2-1731526985115 (Datanode Uuid c0c64aab-1c9e-455b-90f3-92b4fe9dd18d) service to localhost/127.0.0.1:34445 2024-11-13T19:46:00,039 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T19:46:00,040 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T19:46:00,041 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data1/current/BP-1564430831-172.17.0.2-1731526985115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:46:00,041 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/cluster_e984effb-d1cc-3966-de42-70a8c31e4d8c/data/data2/current/BP-1564430831-172.17.0.2-1731526985115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:46:00,042 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T19:46:00,058 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:46:00,066 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:46:00,066 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:46:00,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:46:00,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c558d8bf-4404-0329-aff2-31e3e0db233b/hadoop.log.dir/,STOPPED} 2024-11-13T19:46:00,101 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T19:46:00,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down